beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

merge master branch

Browse source
This commit is contained in:
Sebastien Blot 2023-10-19 10:51:54 +02:00
parent 2600ffbd19 d2d788c5dc
commit ecbdf2f0e1
No known key found for this signature in database
GPG key ID: DFC2902F40449F6A
230 changed files with 4426 additions and 3970 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.github/workflows/bats-hub.yml vendored
View file

@ -15,7 +15,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.6"]
go-version: ["1.21.3"]
name: "Build + tests"
runs-on: ubuntu-latest
@ -37,7 +37,6 @@ jobs:
uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
- name: "Install bats dependencies"
env:

3
.github/workflows/bats-mysql.yml vendored
View file

@ -14,7 +14,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.6"]
go-version: ["1.21.3"]
name: "Build + tests"
runs-on: ubuntu-latest
@ -44,7 +44,6 @@ jobs:
uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
- name: "Install bats dependencies"
env:

9
.github/workflows/bats-postgres.yml vendored
View file

@ -10,14 +10,14 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.6"]
go-version: ["1.21.3"]
name: "Build + tests"
runs-on: ubuntu-latest
timeout-minutes: 30
services:
database:
image: postgres:15
image: postgres:16
env:
POSTGRES_PASSWORD: "secret"
ports:
@ -30,13 +30,13 @@ jobs:
steps:
- name: "Install pg_dump v15"
- name: "Install pg_dump v16"
# we can remove this when it's released on ubuntu-latest
run: |
sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo tee /etc/apt/trusted.gpg.d/pgdg.asc &>/dev/null
sudo apt update
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install postgresql-client-15
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install postgresql-client-16
- name: "Force machineid"
run: |
@ -53,7 +53,6 @@ jobs:
uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
- name: "Install bats dependencies"
env:

3
.github/workflows/bats-sqlite-coverage.yml vendored
View file

@ -11,7 +11,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.6"]
go-version: ["1.21.3"]
name: "Build + tests"
runs-on: ubuntu-latest
@ -34,7 +34,6 @@ jobs:
uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
- name: "Install bats dependencies"
env:

3
.github/workflows/ci-windows-build-msi.yml vendored
View file

@ -23,7 +23,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.6"]
go-version: ["1.21.3"]
name: Build
runs-on: windows-2019
@ -40,7 +40,6 @@ jobs:
uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
- name: Build
run: make windows_installer BUILD_RE2_WASM=1

18
.github/workflows/codeql-analysis.yml vendored
View file

@ -45,6 +45,9 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
# required to pick up tags for BUILD_VERSION
fetch-depth: 0
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
@ -58,8 +61,8 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v2
# - name: Autobuild
# uses: github/codeql-action/autobuild@v2
# ℹ️ Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
@ -68,9 +71,14 @@ jobs:
# and modify them (or add more) to build your code if your project
# uses a compiled language
#- run: |
# make bootstrap
# make release
- name: "Set up Go"
uses: actions/setup-go@v4
with:
go-version: "1.21.0"
cache-dependency-path: "**/go.sum"
- run: |
make clean build BUILD_RE2_WASM=1
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2

5
.github/workflows/go-tests-windows.yml vendored
View file

@ -22,7 +22,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.6"]
go-version: ["1.21.3"]
name: "Build + tests"
runs-on: windows-2022
@ -39,7 +39,6 @@ jobs:
uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
- name: Build
run: |
@ -61,7 +60,7 @@ jobs:
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
with:
version: v1.51
version: v1.54
args: --issues-exit-code=1 --timeout 10m
only-new-issues: false
# the cache is already managed above, enabling it here

5
.github/workflows/go-tests.yml vendored
View file

@ -34,7 +34,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.6"]
go-version: ["1.21.3"]
name: "Build + tests"
runs-on: ubuntu-latest
@ -120,7 +120,6 @@ jobs:
uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
- name: Build and run tests, static
run: |
@ -145,7 +144,7 @@ jobs:
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
with:
version: v1.51
version: v1.54
args: --issues-exit-code=1 --timeout 10m
only-new-issues: false
# the cache is already managed above, enabling it here

5
.github/workflows/release_publish-package.yml vendored
View file

@ -14,7 +14,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.6"]
go-version: ["1.21.3"]
name: Build and upload binary package
runs-on: ubuntu-latest
@ -30,7 +30,6 @@ jobs:
uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
- name: Build the binaries
run: |
@ -42,4 +41,4 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
tag_name="${GITHUB_REF##*/}"
hub release edit -a crowdsec-release.tgz -a vendor.tgz -m "" "$tag_name"
hub release edit -a crowdsec-release.tgz -a vendor.tgz -a *-vendor.tar.xz -m "" "$tag_name"

7
.gitignore vendored
View file

@ -41,12 +41,7 @@ vendor.tgz
# crowdsec binaries
cmd/crowdsec-cli/cscli
cmd/crowdsec/crowdsec
plugins/notifications/http/notification-http
plugins/notifications/slack/notification-slack
plugins/notifications/splunk/notification-splunk
plugins/notifications/email/notification-email
plugins/notifications/dummy/notification-dummy
plugins/notifications/sentinel/notification-sentinel
cmd/notification-*/notification-*
# Test cache (downloaded files)
.cache

15
.golangci.yml
View file

@ -105,6 +105,7 @@ linters:
# Recommended? (easy)
#
- depguard # Go linter that checks if package imports are in a list of acceptable packages
- dogsled # Checks assignments with too many blank identifiers (e.g. x, _, _, _, := f())
- errchkjson # Checks types passed to the json encoding functions. Reports unsupported types and optionally reports occations, where the check for the returned error can be omitted.
- errorlint # errorlint is a linter for that can be used to find code that will cause problems with the error wrapping scheme introduced in Go 1.13.
@ -121,10 +122,10 @@ linters:
- nosprintfhostport # Checks for misuse of Sprintf to construct a host with port in a URL.
- promlinter # Check Prometheus metrics naming via promlint
- revive # Fast, configurable, extensible, flexible, and beautiful linter for Go. Drop-in replacement of golint.
- tagalign # check that struct tags are well aligned [fast: true, auto-fix: true]
- thelper # thelper detects golang test helpers without t.Helper() call and checks the consistency of test helpers
- wastedassign # wastedassign finds wasted assignment statements.
- wrapcheck # Checks that errors returned from external packages are wrapped
- depguard # Go linter that checks if package imports are in a list of acceptable packages
#
# Recommended? (requires some work)
@ -198,6 +199,18 @@ issues:
- govet
text: "shadow: declaration of \"err\" shadows declaration"
#
# typecheck
#
- linters:
- typecheck
text: "undefined: min"
- linters:
- typecheck
text: "undefined: max"
#
# errcheck
#

12
Dockerfile
View file

@ -1,5 +1,5 @@
# vim: set ft=dockerfile:
ARG GOVERSION=1.20.6
ARG GOVERSION=1.21.3
FROM golang:${GOVERSION}-alpine AS build
@ -52,11 +52,11 @@ FROM slim as plugins
# Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
# The files are here for reference, as users will need to mount a new version to be actually able to use notifications
COPY --from=build /go/src/crowdsec/plugins/notifications/email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/sentinel/sentinel.yaml /staging/etc/crowdsec/notifications/sentinel.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-sentinel/sentinel.yaml /staging/etc/crowdsec/notifications/sentinel.yaml
COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins
FROM slim as geoip

12
Dockerfile.debian
View file

@ -1,5 +1,5 @@
# vim: set ft=dockerfile:
ARG GOVERSION=1.20.6
ARG GOVERSION=1.21.3
FROM golang:${GOVERSION}-bookworm AS build
@ -68,11 +68,11 @@ FROM slim as plugins
# Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
# The files are here for reference, as users will need to mount a new version to be actually able to use notifications
COPY --from=build /go/src/crowdsec/plugins/notifications/email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/sentinel/sentinel.yaml /staging/etc/crowdsec/notifications/sentinel.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-sentinel/sentinel.yaml /staging/etc/crowdsec/notifications/sentinel.yaml
COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins
FROM slim as geoip

99
Makefile
View file

@ -23,11 +23,11 @@ BUILD_RE2_WASM ?= 0
BUILD_STATIC ?= 0
# List of plugins to build
PLUGINS ?= $(patsubst ./plugins/notifications/%,%,$(wildcard ./plugins/notifications/*))
PLUGINS ?= $(patsubst ./cmd/notification-%,%,$(wildcard ./cmd/notification-*))
# Can be overriden, if you can deal with the consequences
BUILD_REQUIRE_GO_MAJOR ?= 1
BUILD_REQUIRE_GO_MINOR ?= 20
BUILD_REQUIRE_GO_MINOR ?= 21
#--------------------------------------
@ -38,7 +38,7 @@ BUILD_CODENAME ?= alphaga
CROWDSEC_FOLDER = ./cmd/crowdsec
CSCLI_FOLDER = ./cmd/crowdsec-cli/
PLUGINS_DIR = ./plugins/notifications
PLUGINS_DIR_PREFIX = ./cmd/notification-
CROWDSEC_BIN = crowdsec$(EXT)
CSCLI_BIN = cscli$(EXT)
@ -64,7 +64,7 @@ bool = $(if $(filter $(call lc, $1),1 yes true),1,0)
#--------------------------------------
#
# Define MAKE_FLAGS and LD_OPTS for the sub-makefiles in cmd/ and plugins/
# Define MAKE_FLAGS and LD_OPTS for the sub-makefiles in cmd/
#
MAKE_FLAGS = --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)"
@ -92,7 +92,6 @@ ifeq ($(PKG_CONFIG),)
endif
ifeq ($(RE2_CHECK),)
# we could detect the platform and suggest the command to install
RE2_FAIL := "libre2-dev is not installed, please install it or set BUILD_RE2_WASM=1 to use the WebAssembly version"
else
# += adds a space that we don't want
@ -101,6 +100,7 @@ LD_OPTS_VARS += -X '$(GO_MODULE_NAME)/pkg/cwversion.Libre2=C++'
endif
endif
# Build static to avoid the runtime dependency on libre2.so
ifeq ($(call bool,$(BUILD_STATIC)),1)
BUILD_TYPE = static
EXTLDFLAGS := -extldflags '-static'
@ -109,10 +109,19 @@ BUILD_TYPE = dynamic
EXTLDFLAGS :=
endif
export LD_OPTS=-ldflags "-s -w $(EXTLDFLAGS) $(LD_OPTS_VARS)" \
-trimpath -tags $(GO_TAGS)
# Build with debug symbols, and disable optimizations + inlining, to use Delve
ifeq ($(call bool,$(DEBUG)),1)
STRIP_SYMBOLS :=
DISABLE_OPTIMIZATION := -gcflags "-N -l"
else
STRIP_SYMBOLS := -s -w
DISABLE_OPTIMIZATION :=
endif
ifneq (,$(TEST_COVERAGE))
export LD_OPTS=-ldflags "$(STRIP_SYMBOLS) $(EXTLDFLAGS) $(LD_OPTS_VARS)" \
-trimpath -tags $(GO_TAGS) $(DISABLE_OPTIMIZATION)
ifeq ($(call bool,$(TEST_COVERAGE)),1)
LD_OPTS += -cover
endif
@ -135,19 +144,47 @@ ifneq (,$(RE2_CHECK))
else
$(info Fallback to WebAssembly regexp library. To use the C++ version, make sure you have installed libre2-dev and pkg-config.)
endif
ifeq ($(call bool,$(DEBUG)),1)
$(info Building with debug symbols and disabled optimizations)
endif
ifeq ($(call bool,$(TEST_COVERAGE)),1)
$(info Test coverage collection enabled)
endif
$(info )
.PHONY: all
all: clean test build
.PHONY: plugins
plugins:
@$(foreach plugin,$(PLUGINS), \
$(MAKE) -C $(PLUGINS_DIR)/$(plugin) build $(MAKE_FLAGS); \
$(MAKE) -C $(PLUGINS_DIR_PREFIX)$(plugin) build $(MAKE_FLAGS); \
)
# same as "$(MAKE) -f debian/rules clean" but without the dependency on debhelper
.PHONY: clean-debian
clean-debian:
@$(RM) -r debian/crowdsec
@$(RM) -r debian/crowdsec
@$(RM) -r debian/files
@$(RM) -r debian/.debhelper
@$(RM) -r debian/*.substvars
@$(RM) -r debian/*-stamp
.PHONY: clean-rpm
clean-rpm:
@$(RM) -r rpm/BUILD
@$(RM) -r rpm/BUILDROOT
@$(RM) -r rpm/RPMS
@$(RM) -r rpm/SOURCES/*.tar.gz
@$(RM) -r rpm/SRPMS
.PHONY: clean
clean: testclean
clean: clean-debian clean-rpm testclean
@$(MAKE) -C $(CROWDSEC_FOLDER) clean $(MAKE_FLAGS)
@$(MAKE) -C $(CSCLI_FOLDER) clean $(MAKE_FLAGS)
@$(RM) $(CROWDSEC_BIN) $(WIN_IGNORE_ERR)
@ -155,7 +192,7 @@ clean: testclean
@$(RM) *.log $(WIN_IGNORE_ERR)
@$(RM) crowdsec-release.tgz $(WIN_IGNORE_ERR)
@$(foreach plugin,$(PLUGINS), \
$(MAKE) -C $(PLUGINS_DIR)/$(plugin) clean $(MAKE_FLAGS); \
$(MAKE) -C $(PLUGINS_DIR_PREFIX)$(plugin) clean $(MAKE_FLAGS); \
)
.PHONY: cscli
@ -166,6 +203,12 @@ cscli: goversion
crowdsec: goversion
@$(MAKE) -C $(CROWDSEC_FOLDER) build $(MAKE_FLAGS)
.PHONY: notification-email
notification-email: goversion
@$(MAKE) -C cmd/notification-email build $(MAKE_FLAGS)
.PHONY: testclean
testclean: bats-clean
@$(RM) pkg/apiserver/ent $(WIN_IGNORE_ERR)
@ -201,37 +244,17 @@ localstack:
localstack-stop:
docker-compose -f test/localstack/docker-compose.yml down
# list of plugins that contain go.mod
PLUGIN_VENDOR = $(foreach plugin,$(PLUGINS),$(shell if [ -f $(PLUGINS_DIR)/$(plugin)/go.mod ]; then echo $(PLUGINS_DIR)/$(plugin); fi))
# build vendor.tgz to be distributed with the release
.PHONY: vendor
vendor:
$(foreach plugin_dir,$(PLUGIN_VENDOR), \
cd $(plugin_dir) >/dev/null && \
$(GO) mod vendor && \
cd - >/dev/null; \
)
vendor: vendor-remove
$(GO) mod vendor
tar -czf vendor.tgz vendor $(foreach plugin_dir,$(PLUGIN_VENDOR),$(plugin_dir)/vendor)
.PHONY: tidy
tidy:
$(GO) mod tidy
$(foreach plugin_dir,$(PLUGIN_VENDOR), \
cd $(plugin_dir) >/dev/null && \
$(GO) mod tidy && \
cd - >/dev/null; \
)
tar czf vendor.tgz vendor
tar --create --auto-compress --file=$(RELDIR)-vendor.tar.xz vendor
# remove vendor directories and vendor.tgz
.PHONY: vendor-remove
vendor-remove:
$(foreach plugin_dir,$(PLUGIN_VENDOR), \
$(RM) $(plugin_dir)/vendor; \
)
$(RM) vendor vendor.tgz
$(RM) vendor vendor.tgz *-vendor.tar.xz
.PHONY: package
package:
@ -242,9 +265,9 @@ package:
@$(CP) $(CSCLI_FOLDER)/$(CSCLI_BIN) $(RELDIR)/cmd/crowdsec-cli
@$(foreach plugin,$(PLUGINS), \
$(MKDIR) $(RELDIR)/$(PLUGINS_DIR)/$(plugin); \
$(CP) $(PLUGINS_DIR)/$(plugin)/notification-$(plugin)$(EXT) $(RELDIR)/$(PLUGINS_DIR)/$(plugin); \
$(CP) $(PLUGINS_DIR)/$(plugin)/$(plugin).yaml $(RELDIR)/$(PLUGINS_DIR)/$(plugin)/; \
$(MKDIR) $(RELDIR)/$(PLUGINS_DIR_PREFIX)$(plugin); \
$(CP) $(PLUGINS_DIR_PREFIX)$(plugin)/notification-$(plugin)$(EXT) $(RELDIR)/$(PLUGINS_DIR_PREFIX)$(plugin); \
$(CP) $(PLUGINS_DIR_PREFIX)$(plugin)/$(plugin).yaml $(RELDIR)/$(PLUGINS_DIR_PREFIX)$(plugin)/; \
)
@$(CPR) ./config $(RELDIR)

2
azure-pipelines.yml
View file

@ -27,7 +27,7 @@ stages:
- task: GoTool@0
displayName: "Install Go 1.20"
inputs:
version: '1.20.6'
version: '1.21.3'
- pwsh: |
choco install -y make

4
cmd/crowdsec-cli/Makefile
View file

@ -4,10 +4,8 @@ ifeq ($(OS), Windows_NT)
EXT = .exe
endif
# Go parameters
GO = go
GOBUILD = $(GO) build
GOTEST = $(GO) test
BINARY_NAME = cscli$(EXT)
PREFIX ?= "/"
@ -17,7 +15,7 @@ BIN_PREFIX = $(PREFIX)"/usr/local/bin/"
all: clean build
build: clean
$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
.PHONY: install
install: install-conf install-bin

7
cmd/crowdsec-cli/alerts.go
View file

@ -126,6 +126,12 @@ func AlertsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
}
csvwriter.Flush()
} else if csConfig.Cscli.Output == "json" {
if *alerts == nil {
// avoid returning "null" in json
// could be cleaner if we used slice of alerts directly
fmt.Println("[]")
return nil
}
x, _ := json.MarshalIndent(alerts, "", " ")
fmt.Printf("%s", string(x))
} else if csConfig.Cscli.Output == "human" {
@ -208,6 +214,7 @@ func NewAlertsCmd() *cobra.Command {
Short: "Manage alerts",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
Aliases: []string{"alert"},
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
var err error
if err := csConfig.LoadAPIClient(); err != nil {

4
cmd/crowdsec-cli/bouncers.go
View file

@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"io"
"slices"
"strings"
"time"
@ -12,7 +13,6 @@ import (
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"golang.org/x/exp/slices"
middlewares "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1"
"github.com/crowdsecurity/crowdsec/pkg/database"
@ -160,7 +160,7 @@ func runBouncersDelete(cmd *cobra.Command, args []string) error {
func NewBouncersDeleteCmd() *cobra.Command {
cmdBouncersDelete := &cobra.Command{
Use: "delete MyBouncerName",
Short: "delete a single bouncer from the database",
Short: "delete bouncer(s) from the database",
Args: cobra.MinimumNArgs(1),
Aliases: []string{"remove"},
DisableAutoGenTag: true,

47
cmd/crowdsec-cli/capi.go
View file

@ -60,16 +60,16 @@ func NewCapiRegisterCmd() *cobra.Command {
Short: "Register to Central API (CAPI)",
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
var err error
capiUser, err := generateID(capiUserPrefix)
if err != nil {
log.Fatalf("unable to generate machine id: %s", err)
return fmt.Errorf("unable to generate machine id: %s", err)
}
password := strfmt.Password(generatePassword(passwordLength))
apiurl, err := url.Parse(types.CAPIBaseURL)
if err != nil {
log.Fatalf("unable to parse api url %s : %s", types.CAPIBaseURL, err)
return fmt.Errorf("unable to parse api url %s: %w", types.CAPIBaseURL, err)
}
_, err = apiclient.RegisterClient(&apiclient.Config{
MachineID: capiUser,
@ -80,7 +80,7 @@ func NewCapiRegisterCmd() *cobra.Command {
}, nil)
if err != nil {
log.Fatalf("api client register ('%s'): %s", types.CAPIBaseURL, err)
return fmt.Errorf("api client register ('%s'): %w", types.CAPIBaseURL, err)
}
log.Printf("Successfully registered to Central API (CAPI)")
@ -103,12 +103,12 @@ func NewCapiRegisterCmd() *cobra.Command {
}
apiConfigDump, err := yaml.Marshal(apiCfg)
if err != nil {
log.Fatalf("unable to marshal api credentials: %s", err)
return fmt.Errorf("unable to marshal api credentials: %w", err)
}
if dumpFile != "" {
err = os.WriteFile(dumpFile, apiConfigDump, 0600)
if err != nil {
log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
return fmt.Errorf("write api credentials in '%s' failed: %w", dumpFile, err)
}
log.Printf("Central API credentials dumped to '%s'", dumpFile)
} else {
@ -116,6 +116,8 @@ func NewCapiRegisterCmd() *cobra.Command {
}
log.Warning(ReloadMessage())
return nil
},
}
cmdCapiRegister.Flags().StringVarP(&outputFile, "file", "f", "", "output file destination")
@ -133,53 +135,56 @@ func NewCapiStatusCmd() *cobra.Command {
Short: "Check status with the Central API (CAPI)",
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
if csConfig.API.Server.OnlineClient == nil {
log.Fatalf("Please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
return fmt.Errorf("please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
}
if csConfig.API.Server.OnlineClient.Credentials == nil {
log.Fatalf("no credentials for Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
return fmt.Errorf("no credentials for Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
}
password := strfmt.Password(csConfig.API.Server.OnlineClient.Credentials.Password)
apiurl, err := url.Parse(csConfig.API.Server.OnlineClient.Credentials.URL)
if err != nil {
log.Fatalf("parsing api url ('%s'): %s", csConfig.API.Server.OnlineClient.Credentials.URL, err)
return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err)
}
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
if err := require.Hub(csConfig); err != nil {
return err
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to load hub index : %s", err)
}
scenarios, err := cwhub.GetInstalledScenariosAsString()
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
if err != nil {
log.Fatalf("failed to get scenarios : %s", err)
return fmt.Errorf("failed to get scenarios: %w", err)
}
if len(scenarios) == 0 {
log.Fatalf("no scenarios installed, abort")
return fmt.Errorf("no scenarios installed, abort")
}
Client, err = apiclient.NewDefaultClient(apiurl, CAPIURLPrefix, fmt.Sprintf("crowdsec/%s", version.String()), nil)
if err != nil {
log.Fatalf("init default client: %s", err)
return fmt.Errorf("init default client: %w", err)
}
t := models.WatcherAuthRequest{
MachineID: &csConfig.API.Server.OnlineClient.Credentials.Login,
Password: &password,
Scenarios: scenarios,
}
log.Infof("Loaded credentials from %s", csConfig.API.Server.OnlineClient.CredentialsFilePath)
log.Infof("Trying to authenticate with username %s on %s", csConfig.API.Server.OnlineClient.Credentials.Login, apiurl)
_, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t)
if err != nil {
log.Fatalf("Failed to authenticate to Central API (CAPI) : %s", err)
return fmt.Errorf("failed to authenticate to Central API (CAPI): %w", err)
}
log.Infof("You can successfully interact with Central API (CAPI)")
return nil
},
}

37
cmd/crowdsec-cli/collections.go
View file

@ -7,6 +7,7 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
@ -20,20 +21,8 @@ func NewCollectionsCmd() *cobra.Command {
Aliases: []string{"collection"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if csConfig.Hub == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to get Hub index : %v", err)
if err := require.Hub(csConfig); err != nil {
return err
}
return nil
@ -47,6 +36,7 @@ func NewCollectionsCmd() *cobra.Command {
}
var ignoreError bool
var cmdCollectionsInstall = &cobra.Command{
Use: "install collection",
Short: "Install given collection(s)",
@ -57,7 +47,7 @@ func NewCollectionsCmd() *cobra.Command {
return compAllItems(cwhub.COLLECTIONS, args, toComplete)
},
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
for _, name := range args {
t := cwhub.GetItem(cwhub.COLLECTIONS, name)
if t == nil {
@ -67,11 +57,12 @@ func NewCollectionsCmd() *cobra.Command {
}
if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
if !ignoreError {
log.Fatalf("Error while installing '%s': %s", name, err)
return fmt.Errorf("error while installing '%s': %w", name, err)
}
log.Errorf("Error while installing '%s': %s", name, err)
}
}
return nil
},
}
cmdCollectionsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
@ -89,21 +80,21 @@ func NewCollectionsCmd() *cobra.Command {
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, forceAction)
return
return nil
}
if len(args) == 0 {
log.Fatal("Specify at least one collection to remove or '--all' flag.")
return fmt.Errorf("specify at least one collection to remove or '--all'")
}
for _, name := range args {
if !forceAction {
item := cwhub.GetItem(cwhub.COLLECTIONS, name)
if item == nil {
log.Fatalf("unable to retrieve: %s\n", name)
return fmt.Errorf("unable to retrieve: %s", name)
}
if len(item.BelongsToCollections) > 0 {
log.Warningf("%s belongs to other collections :\n%s\n", name, item.BelongsToCollections)
@ -113,6 +104,7 @@ func NewCollectionsCmd() *cobra.Command {
}
cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, forceAction)
}
return nil
},
}
cmdCollectionsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
@ -129,17 +121,18 @@ func NewCollectionsCmd() *cobra.Command {
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
} else {
if len(args) == 0 {
log.Fatalf("no target collection to upgrade")
return fmt.Errorf("specify at least one collection to upgrade or '--all'")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, forceAction)
}
}
return nil
},
}
cmdCollectionsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the collections")

80
cmd/crowdsec-cli/config_backup.go
View file

@ -1,6 +1,7 @@
package main
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
@ -9,9 +10,76 @@ import (
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
/* Backup crowdsec configurations to directory <dirPath> :
func backupHub(dirPath string) error {
var err error
var itemDirectory string
var upstreamParsers []string
for _, itemType := range cwhub.ItemTypes {
clog := log.WithFields(log.Fields{
"type": itemType,
})
itemMap := cwhub.GetItemMap(itemType)
if itemMap == nil {
clog.Infof("No %s to backup.", itemType)
continue
}
itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
}
upstreamParsers = []string{}
for k, v := range itemMap {
clog = clog.WithFields(log.Fields{
"file": v.Name,
})
if !v.Installed { //only backup installed ones
clog.Debugf("[%s] : not installed", k)
continue
}
//for the local/tainted ones, we backup the full file
if v.Tainted || v.Local || !v.UpToDate {
//we need to backup stages for parsers
if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
}
}
clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
if err = CopyFile(v.LocalPath, tfile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
}
clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
continue
}
clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
upstreamParsers = append(upstreamParsers, v.Name)
}
//write the upstream items
upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
if err != nil {
return fmt.Errorf("failed marshaling upstream parsers : %s", err)
}
err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
if err != nil {
return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
}
clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
}
return nil
}
/*
Backup crowdsec configurations to directory <dirPath>:
- Main config (config.yaml)
- Profiles config (profiles.yaml)
@ -19,6 +87,7 @@ import (
- Backup of API credentials (local API and online API)
- List of scenarios, parsers, postoverflows and collections that are up-to-date
- Tainted/local/out-of-date scenarios, parsers, postoverflows and collections
- Acquisition files (acquis.yaml, acquis.d/*.yaml)
*/
func backupConfigToDirectory(dirPath string) error {
var err error
@ -120,7 +189,7 @@ func backupConfigToDirectory(dirPath string) error {
log.Infof("Saved profiles to %s", backupProfiles)
}
if err = BackupHub(dirPath); err != nil {
if err = backupHub(dirPath); err != nil {
return fmt.Errorf("failed to backup hub config: %s", err)
}
@ -128,15 +197,10 @@ func backupConfigToDirectory(dirPath string) error {
}
func runConfigBackup(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadHub(); err != nil {
if err := require.Hub(csConfig); err != nil {
return err
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
return fmt.Errorf("failed to get Hub index: %w", err)
}
if err := backupConfigToDirectory(args[0]); err != nil {
return fmt.Errorf("failed to backup config: %w", err)
}

11
cmd/crowdsec-cli/config_feature_flags.go
View file

@ -2,10 +2,12 @@ package main
import (
"fmt"
"path/filepath"
"github.com/fatih/color"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
)
@ -87,7 +89,14 @@ func runConfigFeatureFlags(cmd *cobra.Command, args []string) error {
fmt.Println("To enable a feature you can: ")
fmt.Println(" - set the environment variable CROWDSEC_FEATURE_<uppercase_feature_name> to true")
fmt.Printf(" - add the line '- <feature_name>' to the file %s/feature.yaml\n", csConfig.ConfigPaths.ConfigDir)
featurePath, err := filepath.Abs(csconfig.GetFeatureFilePath(ConfigFilePath))
if err != nil {
// we already read the file, shouldn't happen
return err
}
fmt.Printf(" - add the line '- <feature_name>' to the file %s\n", featurePath)
fmt.Println()
if len(enabled) == 0 && len(disabled) == 0 {

134
cmd/crowdsec-cli/config_restore.go
View file

@ -11,6 +11,7 @@ import (
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
@ -20,7 +21,126 @@ type OldAPICfg struct {
Password string `json:"password"`
}
/* Restore crowdsec configurations to directory <dirPath> :
// it's a rip of the cli version, but in silent-mode
func silentInstallItem(name string, obtype string) (string, error) {
var item = cwhub.GetItem(obtype, name)
if item == nil {
return "", fmt.Errorf("error retrieving item")
}
if downloadOnly && item.Downloaded && item.UpToDate {
return fmt.Sprintf("%s is already downloaded and up-to-date", item.Name), nil
}
err := cwhub.DownloadLatest(csConfig.Hub, item, forceAction, false)
if err != nil {
return "", fmt.Errorf("error while downloading %s : %v", item.Name, err)
}
if err := cwhub.AddItem(obtype, *item); err != nil {
return "", err
}
if downloadOnly {
return fmt.Sprintf("Downloaded %s to %s", item.Name, csConfig.Cscli.HubDir+"/"+item.RemotePath), nil
}
err = cwhub.EnableItem(csConfig.Hub, item)
if err != nil {
return "", fmt.Errorf("error while enabling %s : %v", item.Name, err)
}
if err := cwhub.AddItem(obtype, *item); err != nil {
return "", err
}
return fmt.Sprintf("Enabled %s", item.Name), nil
}
func restoreHub(dirPath string) error {
var err error
if err := csConfig.LoadHub(); err != nil {
return err
}
cwhub.SetHubBranch()
for _, itype := range cwhub.ItemTypes {
itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
if _, err = os.Stat(itemDirectory); err != nil {
log.Infof("no %s in backup", itype)
continue
}
/*restore the upstream items*/
upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
file, err := os.ReadFile(upstreamListFN)
if err != nil {
return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
}
var upstreamList []string
err = json.Unmarshal(file, &upstreamList)
if err != nil {
return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
}
for _, toinstall := range upstreamList {
label, err := silentInstallItem(toinstall, itype)
if err != nil {
log.Errorf("Error while installing %s : %s", toinstall, err)
} else if label != "" {
log.Infof("Installed %s : %s", toinstall, label)
} else {
log.Printf("Installed %s : ok", toinstall)
}
}
/*restore the local and tainted items*/
files, err := os.ReadDir(itemDirectory)
if err != nil {
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
}
for _, file := range files {
//this was the upstream data
if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
continue
}
if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
//we expect a stage here
if !file.IsDir() {
continue
}
stage := file.Name()
stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
}
/*find items*/
ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/")
if err != nil {
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
}
//finally copy item
for _, tfile := range ifiles {
log.Infof("Going to restore local/tainted [%s]", tfile.Name())
sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
if err = CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
}
} else {
log.Infof("Going to restore local/tainted [%s]", file.Name())
sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
if err = CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
}
}
}
return nil
}
/*
Restore crowdsec configurations to directory <dirPath>:
- Main config (config.yaml)
- Profiles config (profiles.yaml)
@ -28,6 +148,7 @@ type OldAPICfg struct {
- Backup of API credentials (local API and online API)
- List of scenarios, parsers, postoverflows and collections that are up-to-date
- Tainted/local/out-of-date scenarios, parsers, postoverflows and collections
- Acquisition files (acquis.yaml, acquis.d/*.yaml)
*/
func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
var err error
@ -111,7 +232,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
/*if there is a acquisition dir, restore its content*/
if csConfig.Crowdsec.AcquisitionDirPath != "" {
if err = os.Mkdir(csConfig.Crowdsec.AcquisitionDirPath, 0o700); err != nil {
if err = os.MkdirAll(csConfig.Crowdsec.AcquisitionDirPath, 0o700); err != nil {
return fmt.Errorf("error while creating %s : %s", csConfig.Crowdsec.AcquisitionDirPath, err)
}
}
@ -166,7 +287,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
}
}
if err = RestoreHub(dirPath); err != nil {
if err = restoreHub(dirPath); err != nil {
return fmt.Errorf("failed to restore hub config : %s", err)
}
@ -181,15 +302,10 @@ func runConfigRestore(cmd *cobra.Command, args []string) error {
return err
}
if err := csConfig.LoadHub(); err != nil {
if err := require.Hub(csConfig); err != nil {
return err
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
return fmt.Errorf("failed to get Hub index: %w", err)
}
if err := restoreConfigFromDirectory(args[0], oldBackup); err != nil {
return fmt.Errorf("failed to restore config from %s: %w", args[0], err)
}

7
cmd/crowdsec-cli/config_show.go
View file

@ -57,7 +57,6 @@ func showConfigKey(key string) error {
var configShowTemplate = `Global:
{{- if .ConfigPaths }}
- Configuration Folder : {{.ConfigPaths.ConfigDir}}
- Configuration Folder : {{.ConfigPaths.ConfigDir}}
- Data Folder : {{.ConfigPaths.DataDir}}
- Hub Folder : {{.ConfigPaths.HubDir}}
@ -164,6 +163,12 @@ Central API:
- User : {{.DbConfig.User}}
- DB Name : {{.DbConfig.DbName}}
{{- end }}
{{- if .DbConfig.MaxOpenConns }}
- Max Open Conns : {{.DbConfig.MaxOpenConns}}
{{- end }}
{{- if ne .DbConfig.DecisionBulkSize 0 }}
- Decision Bulk Size : {{.DbConfig.DecisionBulkSize}}
{{- end }}
{{- if .DbConfig.Flush }}
{{- if .DbConfig.Flush.MaxAge }}
- Flush age : {{.DbConfig.Flush.MaxAge}}

130
cmd/crowdsec-cli/console.go
View file

@ -39,7 +39,7 @@ func NewConsoleCmd() *cobra.Command {
if err := require.CAPI(csConfig); err != nil {
return err
}
if err := require.Enrolled(csConfig); err != nil {
if err := require.CAPIRegistered(csConfig); err != nil {
return err
}
return nil
@ -64,25 +64,20 @@ After running this command your will need to validate the enrollment in the weba
`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
password := strfmt.Password(csConfig.API.Server.OnlineClient.Credentials.Password)
apiURL, err := url.Parse(csConfig.API.Server.OnlineClient.Credentials.URL)
if err != nil {
log.Fatalf("Could not parse CAPI URL : %s", err)
return fmt.Errorf("could not parse CAPI URL: %s", err)
}
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
if err := require.Hub(csConfig); err != nil {
return err
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to load hub index : %s", err)
log.Info("Run 'sudo cscli hub update' to get the hub index")
}
scenarios, err := cwhub.GetInstalledScenariosAsString()
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
if err != nil {
log.Fatalf("failed to get scenarios : %s", err)
return fmt.Errorf("failed to get installed scenarios: %s", err)
}
if len(scenarios) == 0 {
@ -99,20 +94,21 @@ After running this command your will need to validate the enrollment in the weba
})
resp, err := c.Auth.EnrollWatcher(context.Background(), args[0], name, tags, overwrite)
if err != nil {
log.Fatalf("Could not enroll instance: %s", err)
return fmt.Errorf("could not enroll instance: %s", err)
}
if resp.Response.StatusCode == 200 && !overwrite {
log.Warning("Instance already enrolled. You can use '--overwrite' to force enroll")
return
return nil
}
SetConsoleOpts(csconfig.CONSOLE_CONFIGS, true)
if err := csConfig.API.Server.DumpConsoleConfig(); err != nil {
log.Fatalf("failed writing console config : %s", err)
if err := SetConsoleOpts([]string{csconfig.SEND_MANUAL_SCENARIOS, csconfig.SEND_TAINTED_SCENARIOS}, true); err != nil {
return err
}
log.Infof("Enabled tainted&manual alerts sharing, see 'cscli console status'.")
log.Infof("Watcher successfully enrolled. Visit https://app.crowdsec.net to accept it.")
log.Infof("Please restart crowdsec after accepting the enrollment.")
log.Info("Enabled tainted&manual alerts sharing, see 'cscli console status'.")
log.Info("Watcher successfully enrolled. Visit https://app.crowdsec.net to accept it.")
log.Info("Please restart crowdsec after accepting the enrollment.")
return nil
},
}
cmdEnroll.Flags().StringVarP(&name, "name", "n", "", "Name to display in the console")
@ -130,21 +126,23 @@ After running this command your will need to validate the enrollment in the weba
Enable given information push to the central API. Allows to empower the console`,
ValidArgs: csconfig.CONSOLE_CONFIGS,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
if enableAll {
SetConsoleOpts(csconfig.CONSOLE_CONFIGS, true)
if err := SetConsoleOpts(csconfig.CONSOLE_CONFIGS, true); err != nil {
return err
}
log.Infof("All features have been enabled successfully")
} else {
if len(args) == 0 {
log.Fatalf("You must specify at least one feature to enable")
return fmt.Errorf("you must specify at least one feature to enable")
}
if err := SetConsoleOpts(args, true); err != nil {
return err
}
SetConsoleOpts(args, true)
log.Infof("%v have been enabled", args)
}
if err := csConfig.API.Server.DumpConsoleConfig(); err != nil {
log.Fatalf("failed writing console config : %s", err)
}
log.Infof(ReloadMessage())
return nil
},
}
cmdEnable.Flags().BoolVarP(&enableAll, "all", "a", false, "Enable all console options")
@ -157,49 +155,55 @@ Enable given information push to the central API. Allows to empower the console`
Long: `
Disable given information push to the central API.`,
ValidArgs: csconfig.CONSOLE_CONFIGS,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if disableAll {
SetConsoleOpts(csconfig.CONSOLE_CONFIGS, false)
} else {
SetConsoleOpts(args, false)
}
if err := csConfig.API.Server.DumpConsoleConfig(); err != nil {
log.Fatalf("failed writing console config : %s", err)
}
RunE: func(cmd *cobra.Command, args []string) error {
if disableAll {
if err := SetConsoleOpts(csconfig.CONSOLE_CONFIGS, false); err != nil {
return err
}
log.Infof("All features have been disabled")
} else {
if err := SetConsoleOpts(args, false); err != nil {
return err
}
log.Infof("%v have been disabled", args)
}
log.Infof(ReloadMessage())
return nil
},
}
cmdDisable.Flags().BoolVarP(&disableAll, "all", "a", false, "Disable all console options")
cmdConsole.AddCommand(cmdDisable)
cmdConsoleStatus := &cobra.Command{
Use: "status [option]",
Short: "Shows status of one or all console options",
Use: "status",
Short: "Shows status of the console options",
Example: `sudo cscli console status`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
switch csConfig.Cscli.Output {
case "human":
cmdConsoleStatusTable(color.Output, *csConfig)
case "json":
data, err := json.MarshalIndent(csConfig.API.Server.ConsoleConfig, "", " ")
if err != nil {
log.Fatalf("failed to marshal configuration: %s", err)
c := csConfig.API.Server.ConsoleConfig
out := map[string](*bool){
csconfig.SEND_MANUAL_SCENARIOS: c.ShareManualDecisions,
csconfig.SEND_CUSTOM_SCENARIOS: c.ShareCustomScenarios,
csconfig.SEND_TAINTED_SCENARIOS: c.ShareTaintedScenarios,
csconfig.SEND_CONTEXT: c.ShareContext,
csconfig.CONSOLE_MANAGEMENT: c.ConsoleManagement,
}
fmt.Printf("%s\n", string(data))
data, err := json.MarshalIndent(out, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal configuration: %s", err)
}
fmt.Println(string(data))
case "raw":
csvwriter := csv.NewWriter(os.Stdout)
err := csvwriter.Write([]string{"option", "enabled"})
if err != nil {
log.Fatal(err)
return err
}
rows := [][]string{
@ -212,11 +216,12 @@ Disable given information push to the central API.`,
for _, row := range rows {
err = csvwriter.Write(row)
if err != nil {
log.Fatal(err)
return err
}
}
csvwriter.Flush()
}
return nil
},
}
cmdConsole.AddCommand(cmdConsoleStatus)
@ -224,7 +229,25 @@ Disable given information push to the central API.`,
return cmdConsole
}
func SetConsoleOpts(args []string, wanted bool) {
func dumpConsoleConfig(c *csconfig.LocalApiServerCfg) error {
out, err := yaml.Marshal(c.ConsoleConfig)
if err != nil {
return fmt.Errorf("while marshaling ConsoleConfig (for %s): %w", c.ConsoleConfigPath, err)
}
if c.ConsoleConfigPath == "" {
c.ConsoleConfigPath = csconfig.DefaultConsoleConfigFilePath
log.Debugf("Empty console_path, defaulting to %s", c.ConsoleConfigPath)
}
if err := os.WriteFile(c.ConsoleConfigPath, out, 0600); err != nil {
return fmt.Errorf("while dumping console config to %s: %w", c.ConsoleConfigPath, err)
}
return nil
}
func SetConsoleOpts(args []string, wanted bool) error {
for _, arg := range args {
switch arg {
case csconfig.CONSOLE_MANAGEMENT:
@ -255,12 +278,12 @@ func SetConsoleOpts(args []string, wanted bool) {
if changed {
fileContent, err := yaml.Marshal(csConfig.API.Server.OnlineClient.Credentials)
if err != nil {
log.Fatalf("Cannot marshal credentials: %s", err)
return fmt.Errorf("cannot marshal credentials: %s", err)
}
log.Infof("Updating credentials file: %s", csConfig.API.Server.OnlineClient.CredentialsFilePath)
err = os.WriteFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, fileContent, 0600)
if err != nil {
log.Fatalf("Cannot write credentials file: %s", err)
return fmt.Errorf("cannot write credentials file: %s", err)
}
}
}
@ -317,8 +340,13 @@ func SetConsoleOpts(args []string, wanted bool) {
csConfig.API.Server.ConsoleConfig.ShareContext = ptr.Of(wanted)
}
default:
log.Fatalf("unknown flag %s", arg)
return fmt.Errorf("unknown flag %s", arg)
}
}
if err := dumpConsoleConfig(csConfig.API.Server); err != nil {
return fmt.Errorf("failed writing console config: %s", err)
}
return nil
}

6
cmd/crowdsec-cli/decisions.go
View file

@ -81,6 +81,12 @@ func DecisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error
}
csvwriter.Flush()
} else if csConfig.Cscli.Output == "json" {
if *alerts == nil {
// avoid returning "null" in `json"
// could be cleaner if we used slice of alerts directly
fmt.Println("[]")
return nil
}
x, _ := json.MarshalIndent(alerts, "", " ")
fmt.Printf("%s", string(x))
} else if csConfig.Cscli.Output == "human" {

17
cmd/crowdsec-cli/decisions_import.go
View file

@ -188,7 +188,9 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error {
}
}
alerts := models.AddAlertsRequest{}
if len(decisions) > 1000 {
log.Infof("You are about to add %d decisions, this may take a while", len(decisions))
}
for _, chunk := range slicetools.Chunks(decisions, batchSize) {
log.Debugf("Processing chunk of %d decisions", len(chunk))
@ -212,16 +214,11 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error {
ScenarioVersion: ptr.Of(""),
Decisions: chunk,
}
alerts = append(alerts, &importAlert)
}
if len(decisions) > 1000 {
log.Infof("You are about to add %d decisions, this may take a while", len(decisions))
}
_, _, err = Client.Alerts.Add(context.Background(), alerts)
if err != nil {
return err
_, _, err = Client.Alerts.Add(context.Background(), models.AddAlertsRequest{&importAlert})
if err != nil {
return err
}
}
log.Infof("Imported %d decisions", len(decisions))

32
cmd/crowdsec-cli/explain.go
View file

@ -12,9 +12,22 @@ import (
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/hubtest"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
func GetLineCountForFile(filepath string) (int, error) {
f, err := os.Open(filepath)
if err != nil {
return 0, err
}
defer f.Close()
lc := 0
fs := bufio.NewScanner(f)
for fs.Scan() {
lc++
}
return lc, nil
}
func runExplain(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
@ -61,6 +74,11 @@ func runExplain(cmd *cobra.Command, args []string) error {
return err
}
labels, err := flags.GetString("labels")
if err != nil {
return err
}
fileInfo, _ := os.Stdin.Stat()
if logType == "" || (logLine == "" && logFile == "" && dsn == "") {
@ -123,9 +141,12 @@ func runExplain(cmd *cobra.Command, args []string) error {
return fmt.Errorf("unable to get absolute path of '%s', exiting", logFile)
}
dsn = fmt.Sprintf("file://%s", absolutePath)
lineCount := types.GetLineCountForFile(absolutePath)
lineCount, err := GetLineCountForFile(absolutePath)
if err != nil {
return err
}
if lineCount > 100 {
log.Warnf("log file contains %d lines. This may take lot of resources.", lineCount)
log.Warnf("The log file contains %d lines. This may take a lot of resources.", lineCount)
}
}
@ -134,6 +155,10 @@ func runExplain(cmd *cobra.Command, args []string) error {
}
cmdArgs := []string{"-c", ConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", dir, "-no-api"}
if labels != "" {
log.Debugf("adding labels %s", labels)
cmdArgs = append(cmdArgs, "-label", labels)
}
crowdsecCmd := exec.Command(crowdsec, cmdArgs...)
output, err := crowdsecCmd.CombinedOutput()
if err != nil {
@ -193,6 +218,7 @@ tail -n 5 myfile.log | cscli explain --type nginx -f -
flags.StringP("dsn", "d", "", "DSN to test")
flags.StringP("log", "l", "", "Log line to test")
flags.StringP("type", "t", "", "Type of the acquisition to test")
flags.String("labels", "", "Additional labels to add to the acquisition format (key:value,key2:value2)")
flags.BoolP("verbose", "v", false, "Display individual changes")
flags.Bool("failures", false, "Only show failed lines")
flags.Bool("only-successful-parsers", false, "Only show successful parsers")

66
cmd/crowdsec-cli/hub.go
View file

@ -8,6 +8,7 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
@ -50,17 +51,13 @@ func NewHubListCmd() *cobra.Command {
Short: "List installed configs",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
return err
}
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to get Hub index : %v", err)
}
//use LocalSync to get warnings about tainted / outdated items
_, warn := cwhub.LocalSync(csConfig.Hub)
// use LocalSync to get warnings about tainted / outdated items
warn, _ := cwhub.LocalSync(csConfig.Hub)
for _, v := range warn {
log.Info(v)
}
@ -68,6 +65,8 @@ func NewHubListCmd() *cobra.Command {
ListItems(color.Output, []string{
cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW,
}, args, true, false, all)
return nil
},
}
cmdHubList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
@ -89,31 +88,31 @@ Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.inde
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
cwhub.SetHubBranch()
return nil
},
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
return err
}
if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
if errors.Is(err, cwhub.ErrIndexNotFound) {
log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch)
cwhub.HubBranch = "master"
if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to get Hub index after retry : %v", err)
}
} else {
log.Fatalf("Failed to get Hub index : %v", err)
if !errors.Is(err, cwhub.ErrIndexNotFound) {
return fmt.Errorf("failed to get Hub index : %w", err)
}
log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch)
cwhub.HubBranch = "master"
if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
return fmt.Errorf("failed to get Hub index after retry: %w", err)
}
}
//use LocalSync to get warnings about tainted / outdated items
_, warn := cwhub.LocalSync(csConfig.Hub)
// use LocalSync to get warnings about tainted / outdated items
warn, _ := cwhub.LocalSync(csConfig.Hub)
for _, v := range warn {
log.Info(v)
}
return nil
},
}
@ -134,18 +133,13 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
cwhub.SetHubBranch()
return nil
},
Run: func(cmd *cobra.Command, args []string) {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to get Hub index : %v", err)
RunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
return err
}
log.Infof("Upgrading collections")
@ -156,6 +150,8 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if
cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
log.Infof("Upgrading postoverflows")
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
return nil
},
}
cmdHubUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")

64
cmd/crowdsec-cli/lapi.go
View file

@ -5,17 +5,18 @@ import (
"fmt"
"net/url"
"os"
"slices"
"sort"
"strings"
"github.com/go-openapi/strfmt"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"golang.org/x/exp/slices"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
@ -36,15 +37,12 @@ func runLapiStatus(cmd *cobra.Command, args []string) error {
if err != nil {
log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
}
if err := csConfig.LoadHub(); err != nil {
if err := require.Hub(csConfig); err != nil {
log.Fatal(err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to load hub index : %s", err)
}
scenarios, err := cwhub.GetInstalledScenariosAsString()
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
if err != nil {
log.Fatalf("failed to get scenarios : %s", err)
}
@ -216,6 +214,29 @@ func NewLapiCmd() *cobra.Command {
return cmdLapi
}
func AddContext(key string, values []string) error {
if err := alertcontext.ValidateContextExpr(key, values); err != nil {
return fmt.Errorf("invalid context configuration :%s", err)
}
if _, ok := csConfig.Crowdsec.ContextToSend[key]; !ok {
csConfig.Crowdsec.ContextToSend[key] = make([]string, 0)
log.Infof("key '%s' added", key)
}
data := csConfig.Crowdsec.ContextToSend[key]
for _, val := range values {
if !slices.Contains(data, val) {
log.Infof("value '%s' added to key '%s'", val, key)
data = append(data, val)
}
csConfig.Crowdsec.ContextToSend[key] = data
}
if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
return err
}
return nil
}
func NewLapiContextCmd() *cobra.Command {
cmdContext := &cobra.Command{
Use: "context [command]",
@ -246,32 +267,29 @@ func NewLapiContextCmd() *cobra.Command {
Short: "Add context to send with alerts. You must specify the output key with the expr value you want",
Example: `cscli lapi context add --key source_ip --value evt.Meta.source_ip
cscli lapi context add --key file_source --value evt.Line.Src
cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if err := alertcontext.ValidateContextExpr(keyToAdd, valuesToAdd); err != nil {
log.Fatalf("invalid context configuration :%s", err)
}
if _, ok := csConfig.Crowdsec.ContextToSend[keyToAdd]; !ok {
csConfig.Crowdsec.ContextToSend[keyToAdd] = make([]string, 0)
log.Infof("key '%s' added", keyToAdd)
}
data := csConfig.Crowdsec.ContextToSend[keyToAdd]
for _, val := range valuesToAdd {
if !slices.Contains(data, val) {
log.Infof("value '%s' added to key '%s'", val, keyToAdd)
data = append(data, val)
if keyToAdd != "" {
if err := AddContext(keyToAdd, valuesToAdd); err != nil {
log.Fatalf(err.Error())
}
csConfig.Crowdsec.ContextToSend[keyToAdd] = data
return
}
if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
log.Fatalf(err.Error())
for _, v := range valuesToAdd {
keySlice := strings.Split(v, ".")
key := keySlice[len(keySlice)-1]
value := []string{v}
if err := AddContext(key, value); err != nil {
log.Fatalf(err.Error())
}
}
},
}
cmdContextAdd.Flags().StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send")
cmdContextAdd.Flags().StringSliceVar(&valuesToAdd, "value", []string{}, "The expr fields to associate with the key")
cmdContextAdd.MarkFlagRequired("key")
cmdContextAdd.MarkFlagRequired("value")
cmdContext.AddCommand(cmdContextAdd)

2
cmd/crowdsec-cli/machines.go
View file

@ -8,6 +8,7 @@ import (
"io"
"math/big"
"os"
"slices"
"strings"
"time"
@ -17,7 +18,6 @@ import (
"github.com/google/uuid"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"golang.org/x/exp/slices"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/machineid"

6
cmd/crowdsec-cli/main.go
View file

@ -4,6 +4,7 @@ import (
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"github.com/fatih/color"
@ -11,7 +12,6 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/cobra/doc"
"golang.org/x/exp/slices"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
@ -53,11 +53,11 @@ func initConfig() {
}
if !slices.Contains(NoNeedConfig, os.Args[1]) {
log.Debugf("Using %s as configuration file", ConfigFilePath)
csConfig, mergedConfig, err = csconfig.NewConfig(ConfigFilePath, false, false, true)
if err != nil {
log.Fatal(err)
}
log.Debugf("Using %s as configuration file", ConfigFilePath)
if err := csConfig.LoadCSCLI(); err != nil {
log.Fatal(err)
}
@ -188,7 +188,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
/*usage*/
var cmdVersion = &cobra.Command{
Use: "version",
Short: "Display version and exit.",
Short: "Display version",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {

56
cmd/crowdsec-cli/parsers.go
View file

@ -7,10 +7,10 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewParsersCmd() *cobra.Command {
var cmdParsers = &cobra.Command{
Use: "parsers [action] [config]",
@ -25,21 +25,10 @@ cscli parsers remove crowdsecurity/sshd-logs
Aliases: []string{"parser"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if csConfig.Hub == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
if err := require.Hub(csConfig); err != nil {
return err
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to get Hub index : %v", err)
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
@ -59,7 +48,6 @@ cscli parsers remove crowdsecurity/sshd-logs
return cmdParsers
}
func NewParsersInstallCmd() *cobra.Command {
var ignoreError bool
@ -73,7 +61,7 @@ func NewParsersInstallCmd() *cobra.Command {
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cwhub.PARSERS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
for _, name := range args {
t := cwhub.GetItem(cwhub.PARSERS, name)
if t == nil {
@ -82,15 +70,16 @@ func NewParsersInstallCmd() *cobra.Command {
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
if ignoreError {
log.Errorf("Error while installing '%s': %s", name, err)
} else {
log.Fatalf("Error while installing '%s': %s", name, err)
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", name, err)
}
log.Errorf("Error while installing '%s': %s", name, err)
}
}
return nil
},
}
cmdParsersInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
cmdParsersInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
cmdParsersInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple parsers")
@ -98,33 +87,35 @@ func NewParsersInstallCmd() *cobra.Command {
return cmdParsersInstall
}
func NewParsersRemoveCmd() *cobra.Command {
var cmdParsersRemove = &cobra.Command{
cmdParsersRemove := &cobra.Command{
Use: "remove [config]",
Short: "Remove given parser(s)",
Long: `Remove given parse(s) from hub`,
Aliases: []string{"delete"},
Example: `cscli parsers remove crowdsec/xxx crowdsec/xyz`,
Aliases: []string{"delete"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, forceAction)
return
return nil
}
if len(args) == 0 {
log.Fatalf("Specify at least one parser to remove or '--all' flag.")
return fmt.Errorf("specify at least one parser to remove or '--all'")
}
for _, name := range args {
cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, forceAction)
}
return nil
},
}
cmdParsersRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
cmdParsersRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
cmdParsersRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the parsers")
@ -132,9 +123,8 @@ func NewParsersRemoveCmd() *cobra.Command {
return cmdParsersRemove
}
func NewParsersUpgradeCmd() *cobra.Command {
var cmdParsersUpgrade = &cobra.Command{
cmdParsersUpgrade := &cobra.Command{
Use: "upgrade [config]",
Short: "Upgrade given parser(s)",
Long: `Fetch and upgrade given parser(s) from hub`,
@ -143,26 +133,27 @@ func NewParsersUpgradeCmd() *cobra.Command {
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
} else {
if len(args) == 0 {
log.Fatalf("no target parser to upgrade")
return fmt.Errorf("specify at least one parser to upgrade or '--all'")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, forceAction)
}
}
return nil
},
}
cmdParsersUpgrade.PersistentFlags().BoolVar(&all, "all", false, "Upgrade all the parsers")
cmdParsersUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
return cmdParsersUpgrade
}
func NewParsersInspectCmd() *cobra.Command {
var cmdParsersInspect = &cobra.Command{
Use: "inspect [name]",
@ -178,12 +169,12 @@ func NewParsersInspectCmd() *cobra.Command {
InspectItem(args[0], cwhub.PARSERS)
},
}
cmdParsersInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
return cmdParsersInspect
}
func NewParsersListCmd() *cobra.Command {
var cmdParsersList = &cobra.Command{
Use: "list [name]",
@ -196,6 +187,7 @@ cscli parser list crowdsecurity/xxx`,
ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all)
},
}
cmdParsersList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
return cmdParsersList

113
cmd/crowdsec-cli/postoverflows.go
View file

@ -7,9 +7,46 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewPostOverflowsCmd() *cobra.Command {
cmdPostOverflows := &cobra.Command{
Use: "postoverflows [action] [config]",
Short: "Install/Remove/Upgrade/Inspect postoverflow(s) from hub",
Example: `cscli postoverflows install crowdsecurity/cdn-whitelist
cscli postoverflows inspect crowdsecurity/cdn-whitelist
cscli postoverflows upgrade crowdsecurity/cdn-whitelist
cscli postoverflows list
cscli postoverflows remove crowdsecurity/cdn-whitelist`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"postoverflow"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
return err
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof(ReloadMessage())
},
}
cmdPostOverflows.AddCommand(NewPostOverflowsInstallCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsRemoveCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsUpgradeCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsInspectCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsListCmd())
return cmdPostOverflows
}
func NewPostOverflowsInstallCmd() *cobra.Command {
var ignoreError bool
@ -23,7 +60,7 @@ func NewPostOverflowsInstallCmd() *cobra.Command {
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
for _, name := range args {
t := cwhub.GetItem(cwhub.PARSERS_OVFLW, name)
if t == nil {
@ -32,13 +69,13 @@ func NewPostOverflowsInstallCmd() *cobra.Command {
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil {
if ignoreError {
log.Errorf("Error while installing '%s': %s", name, err)
} else {
log.Fatalf("Error while installing '%s': %s", name, err)
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", name, err)
}
log.Errorf("Error while installing '%s': %s", name, err)
}
}
return nil
},
}
@ -55,24 +92,26 @@ func NewPostOverflowsRemoveCmd() *cobra.Command {
Short: "Remove given postoverflow(s)",
Long: `remove given postoverflow(s)`,
Example: `cscli postoverflows remove crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
Aliases: []string{"delete"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, "", all, purge, forceAction)
return
return nil
}
if len(args) == 0 {
log.Fatalf("Specify at least one postoverflow to remove or '--all' flag.")
return fmt.Errorf("specify at least one postoverflow to remove or '--all'")
}
for _, name := range args {
cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, name, all, purge, forceAction)
}
return nil
},
}
@ -93,17 +132,18 @@ func NewPostOverflowsUpgradeCmd() *cobra.Command {
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
} else {
if len(args) == 0 {
log.Fatalf("no target postoverflow to upgrade")
return fmt.Errorf("specify at least one postoverflow to upgrade or '--all'")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, name, forceAction)
}
}
return nil
},
}
@ -120,10 +160,10 @@ func NewPostOverflowsInspectCmd() *cobra.Command {
Long: `Inspect given postoverflow`,
Example: `cscli postoverflows inspect crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
InspectItem(args[0], cwhub.PARSERS_OVFLW)
},
@ -149,52 +189,3 @@ cscli postoverflows list crowdsecurity/xxx`,
return cmdPostOverflowsList
}
func NewPostOverflowsCmd() *cobra.Command {
cmdPostOverflows := &cobra.Command{
Use: "postoverflows [action] [config]",
Short: "Install/Remove/Upgrade/Inspect postoverflow(s) from hub",
Example: `cscli postoverflows install crowdsecurity/cdn-whitelist
cscli postoverflows inspect crowdsecurity/cdn-whitelist
cscli postoverflows upgrade crowdsecurity/cdn-whitelist
cscli postoverflows list
cscli postoverflows remove crowdsecurity/cdn-whitelist`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"postoverflow"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if csConfig.Hub == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to get Hub index : %v", err)
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof(ReloadMessage())
},
}
cmdPostOverflows.AddCommand(NewPostOverflowsInstallCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsRemoveCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsUpgradeCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsInspectCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsListCmd())
return cmdPostOverflows
}

20
cmd/crowdsec-cli/require/require.go
View file

@ -4,6 +4,7 @@ import (
"fmt"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func LAPI(c *csconfig.Config) error {
@ -32,7 +33,7 @@ func PAPI(c *csconfig.Config) error {
return nil
}
func Enrolled(c *csconfig.Config) error {
func CAPIRegistered(c *csconfig.Config) error {
if c.API.Server.OnlineClient.Credentials == nil {
return fmt.Errorf("the Central API (CAPI) must be configured with 'cscli capi register'")
}
@ -63,3 +64,20 @@ func Notifications(c *csconfig.Config) error {
return nil
}
func Hub (c *csconfig.Config) error {
if err := c.LoadHub(); err != nil {
return err
}
if c.Hub == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
cwhub.SetHubBranch()
if err := cwhub.GetHubIdx(c.Hub); err != nil {
return fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err)
}
return nil
}

39
cmd/crowdsec-cli/scenarios.go
View file

@ -7,6 +7,7 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
@ -24,20 +25,8 @@ cscli scenarios remove crowdsecurity/ssh-bf
Aliases: []string{"scenario"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if csConfig.Hub == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("while setting hub branch: %w", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to get Hub index : %v", err)
if err := require.Hub(csConfig); err != nil {
return err
}
return nil
@ -72,7 +61,7 @@ func NewCmdScenariosInstall() *cobra.Command {
return compAllItems(cwhub.SCENARIOS, args, toComplete)
},
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
for _, name := range args {
t := cwhub.GetItem(cwhub.SCENARIOS, name)
if t == nil {
@ -81,13 +70,13 @@ func NewCmdScenariosInstall() *cobra.Command {
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
if ignoreError {
log.Errorf("Error while installing '%s': %s", name, err)
} else {
log.Fatalf("Error while installing '%s': %s", name, err)
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", name, err)
}
log.Errorf("Error while installing '%s': %s", name, err)
}
}
return nil
},
}
cmdScenariosInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
@ -108,19 +97,20 @@ func NewCmdScenariosRemove() *cobra.Command {
return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
},
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, forceAction)
return
return nil
}
if len(args) == 0 {
log.Fatalf("Specify at least one scenario to remove or '--all' flag.")
return fmt.Errorf("specify at least one scenario to remove or '--all'")
}
for _, name := range args {
cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, forceAction)
}
return nil
},
}
cmdScenariosRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
@ -140,17 +130,18 @@ func NewCmdScenariosUpgrade() *cobra.Command {
return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
},
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
} else {
if len(args) == 0 {
log.Fatalf("no target scenario to upgrade")
return fmt.Errorf("specify at least one scenario to upgrade or '--all'")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, forceAction)
}
}
return nil
},
}
cmdScenariosUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the scenarios")

18
cmd/crowdsec-cli/setup.go
View file

@ -112,6 +112,20 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
return err
}
var detectReader *os.File
switch detectConfigFile {
case "-":
log.Tracef("Reading detection rules from stdin")
detectReader = os.Stdin
default:
log.Tracef("Reading detection rules: %s", detectConfigFile)
detectReader, err = os.Open(detectConfigFile)
if err != nil {
return err
}
}
listSupportedServices, err := flags.GetBool("list-supported-services")
if err != nil {
return err
@ -171,7 +185,7 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
}
if listSupportedServices {
supported, err := setup.ListSupported(detectConfigFile)
supported, err := setup.ListSupported(detectReader)
if err != nil {
return err
}
@ -195,7 +209,7 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
SnubSystemd: snubSystemd,
}
hubSetup, err := setup.Detect(detectConfigFile, opts)
hubSetup, err := setup.Detect(detectReader, opts)
if err != nil {
return fmt.Errorf("detecting services: %w", err)
}

11
cmd/crowdsec-cli/simulation.go
View file

@ -3,12 +3,13 @@ package main
import (
"fmt"
"os"
"slices"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"golang.org/x/exp/slices"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
@ -18,7 +19,7 @@ func addToExclusion(name string) error {
}
func removeFromExclusion(name string) error {
index := indexOf(name, csConfig.Cscli.SimulationConfig.Exclusions)
index := slices.Index(csConfig.Cscli.SimulationConfig.Exclusions, name)
// Remove element from the slice
csConfig.Cscli.SimulationConfig.Exclusions[index] = csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
@ -144,13 +145,9 @@ func NewSimulationEnableCmd() *cobra.Command {
Example: `cscli simulation enable`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if err := csConfig.LoadHub(); err != nil {
if err := require.Hub(csConfig); err != nil {
log.Fatal(err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to get Hub index : %v", err)
}
if len(args) > 0 {
for _, scenario := range args {

24
cmd/crowdsec-cli/support.go
View file

@ -20,6 +20,7 @@ import (
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
@ -131,24 +132,6 @@ func collectOSInfo() ([]byte, error) {
return w.Bytes(), nil
}
func initHub() error {
if err := csConfig.LoadHub(); err != nil {
return fmt.Errorf("cannot load hub: %s", err)
}
if csConfig.Hub == nil {
return fmt.Errorf("hub not configured")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("cannot set hub branch: %s", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
return fmt.Errorf("no hub index found: %s", err)
}
return nil
}
func collectHubItems(itemType string) []byte {
out := bytes.NewBuffer(nil)
log.Infof("Collecting %s list", itemType)
@ -184,7 +167,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
if err != nil {
return []byte(fmt.Sprintf("cannot parse API URL: %s", err))
}
scenarios, err := cwhub.GetInstalledScenariosAsString()
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
if err != nil {
return []byte(fmt.Sprintf("could not collect scenarios: %s", err))
}
@ -312,8 +295,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
skipAgent = true
}
err = initHub()
if err != nil {
if err := require.Hub(csConfig); err != nil {
log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected")
skipHub = true
infos[SUPPORT_PARSERS_PATH] = []byte(err.Error())

240
cmd/crowdsec-cli/utils.go
View file

@ -8,7 +8,7 @@ import (
"math"
"net"
"net/http"
"os"
"slices"
"strconv"
"strings"
"time"
@ -19,11 +19,11 @@ import (
"github.com/prometheus/prom2json"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"golang.org/x/exp/slices"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/types"
@ -38,34 +38,6 @@ func printHelp(cmd *cobra.Command) {
}
}
func indexOf(s string, slice []string) int {
for i, elem := range slice {
if s == elem {
return i
}
}
return -1
}
func LoadHub() error {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if csConfig.Hub == nil {
return fmt.Errorf("unable to load hub")
}
if err := cwhub.SetHubBranch(); err != nil {
log.Warningf("unable to set hub branch (%s), default to master", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
return fmt.Errorf("Failed to get Hub index : '%w'. Run 'sudo cscli hub update' to get the hub index", err)
}
return nil
}
func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) {
errMsg := ""
if score < MaxDistance {
@ -100,7 +72,7 @@ func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
}
func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if err := LoadHub(); err != nil {
if err := require.Hub(csConfig); err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
@ -116,31 +88,16 @@ func compAllItems(itemType string, args []string, toComplete string) ([]string,
}
func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if err := LoadHub(); err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
var items []string
var err error
switch itemType {
case cwhub.PARSERS:
items, err = cwhub.GetInstalledParsersAsString()
case cwhub.SCENARIOS:
items, err = cwhub.GetInstalledScenariosAsString()
case cwhub.PARSERS_OVFLW:
items, err = cwhub.GetInstalledPostOverflowsAsString()
case cwhub.COLLECTIONS:
items, err = cwhub.GetInstalledCollectionsAsString()
case cwhub.WAAP_RULES:
items, err = cwhub.GetInstalledWafRulesAsString()
default:
if err := require.Hub(csConfig); err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
items, err := cwhub.GetInstalledItemsAsString(itemType)
if err != nil {
cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
if toComplete != "" {
@ -470,37 +427,6 @@ func GetScenarioMetric(url string, itemName string) map[string]int {
return stats
}
// it's a rip of the cli version, but in silent-mode
func silenceInstallItem(name string, obtype string) (string, error) {
var item = cwhub.GetItem(obtype, name)
if item == nil {
return "", fmt.Errorf("error retrieving item")
}
it := *item
if downloadOnly && it.Downloaded && it.UpToDate {
return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil
}
it, err := cwhub.DownloadLatest(csConfig.Hub, it, forceAction, false)
if err != nil {
return "", fmt.Errorf("error while downloading %s : %v", it.Name, err)
}
if err := cwhub.AddItem(obtype, it); err != nil {
return "", err
}
if downloadOnly {
return fmt.Sprintf("Downloaded %s to %s", it.Name, csConfig.Cscli.HubDir+"/"+it.RemotePath), nil
}
it, err = cwhub.EnableItem(csConfig.Hub, it)
if err != nil {
return "", fmt.Errorf("error while enabling %s : %v", it.Name, err)
}
if err := cwhub.AddItem(obtype, it); err != nil {
return "", err
}
return fmt.Sprintf("Enabled %s", it.Name), nil
}
func GetPrometheusMetric(url string) []*prom2json.Family {
mfChan := make(chan *dto.MetricFamily, 1024)
@ -529,160 +455,6 @@ func GetPrometheusMetric(url string) []*prom2json.Family {
return result
}
func RestoreHub(dirPath string) error {
var err error
if err := csConfig.LoadHub(); err != nil {
return err
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
for _, itype := range cwhub.ItemTypes {
itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
if _, err = os.Stat(itemDirectory); err != nil {
log.Infof("no %s in backup", itype)
continue
}
/*restore the upstream items*/
upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
file, err := os.ReadFile(upstreamListFN)
if err != nil {
return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
}
var upstreamList []string
err = json.Unmarshal(file, &upstreamList)
if err != nil {
return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
}
for _, toinstall := range upstreamList {
label, err := silenceInstallItem(toinstall, itype)
if err != nil {
log.Errorf("Error while installing %s : %s", toinstall, err)
} else if label != "" {
log.Infof("Installed %s : %s", toinstall, label)
} else {
log.Printf("Installed %s : ok", toinstall)
}
}
/*restore the local and tainted items*/
files, err := os.ReadDir(itemDirectory)
if err != nil {
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
}
for _, file := range files {
//this was the upstream data
if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
continue
}
if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
//we expect a stage here
if !file.IsDir() {
continue
}
stage := file.Name()
stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
}
/*find items*/
ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/")
if err != nil {
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
}
//finally copy item
for _, tfile := range ifiles {
log.Infof("Going to restore local/tainted [%s]", tfile.Name())
sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
if err = CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
}
} else {
log.Infof("Going to restore local/tainted [%s]", file.Name())
sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
if err = CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
}
}
}
return nil
}
func BackupHub(dirPath string) error {
var err error
var itemDirectory string
var upstreamParsers []string
for _, itemType := range cwhub.ItemTypes {
clog := log.WithFields(log.Fields{
"type": itemType,
})
itemMap := cwhub.GetItemMap(itemType)
if itemMap == nil {
clog.Infof("No %s to backup.", itemType)
continue
}
itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
}
upstreamParsers = []string{}
for k, v := range itemMap {
clog = clog.WithFields(log.Fields{
"file": v.Name,
})
if !v.Installed { //only backup installed ones
clog.Debugf("[%s] : not installed", k)
continue
}
//for the local/tainted ones, we backup the full file
if v.Tainted || v.Local || !v.UpToDate {
//we need to backup stages for parsers
if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
}
}
clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
if err = CopyFile(v.LocalPath, tfile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
}
clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
continue
}
clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
upstreamParsers = append(upstreamParsers, v.Name)
}
//write the upstream items
upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
if err != nil {
return fmt.Errorf("failed marshaling upstream parsers : %s", err)
}
err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
if err != nil {
return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
}
clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
}
return nil
}
type unit struct {
value int64
symbol string

2
cmd/crowdsec-cli/utils_table.go
View file

@ -17,7 +17,7 @@ func listHubItemTable(out io.Writer, title string, statuses []cwhub.ItemHubStatu
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
for _, status := range statuses {
t.AddRow(status.Name, status.UTF8_Status, status.LocalVersion, status.LocalPath)
t.AddRow(status.Name, status.UTF8Status, status.LocalVersion, status.LocalPath)
}
renderTableTitle(out, title)
t.Render()

4
cmd/crowdsec-cli/waap_rules.go
View file

@ -31,9 +31,7 @@ cscli waap-rules remove crowdsecurity/core-rule-set
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
cwhub.SetHubBranch()
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")

3
cmd/crowdsec/Makefile
View file

@ -4,7 +4,6 @@ ifeq ($(OS), Windows_NT)
EXT = .exe
endif
# Go parameters
GO = go
GOBUILD = $(GO) build
GOTEST = $(GO) test
@ -23,7 +22,7 @@ SYSTEMD_PATH_FILE = "/etc/systemd/system/crowdsec.service"
all: clean test build
build: clean
$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(CROWDSEC_BIN)
$(GOBUILD) $(LD_OPTS) -o $(CROWDSEC_BIN)
test:
$(GOTEST) $(LD_OPTS) -v ./...

10
cmd/crowdsec/main.go
View file

@ -138,11 +138,13 @@ func (l *labelsMap) String() string {
}
func (l labelsMap) Set(label string) error {
split := strings.Split(label, ":")
if len(split) != 2 {
return errors.Wrapf(errors.New("Bad Format"), "for Label '%s'", label)
for _, pair := range strings.Split(label, ",") {
split := strings.Split(pair, ":")
if len(split) != 2 {
return fmt.Errorf("invalid format for label '%s', must be key:value", pair)
}
l[split[0]] = split[1]
}
l[split[0]] = split[1]
return nil
}

4
cmd/crowdsec/output.go
View file

@ -70,7 +70,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
var cache []types.RuntimeAlert
var cacheMutex sync.Mutex
scenarios, err := cwhub.GetInstalledScenariosAsString()
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
if err != nil {
return fmt.Errorf("loading list of installed hub scenarios: %w", err)
}
@ -93,7 +93,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
URL: apiURL,
PapiURL: papiURL,
VersionPrefix: "v1",
UpdateScenario: cwhub.GetInstalledScenariosAsString,
UpdateScenario: func() ([]string, error) {return cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)},
})
if err != nil {
return fmt.Errorf("new client api: %w", err)

20
cmd/crowdsec/serve.go
View file

@ -141,12 +141,24 @@ func ShutdownCrowdsecRoutines() error {
time.Sleep(1 * time.Second) // ugly workaround for now
outputsTomb.Kill(nil)
if err := outputsTomb.Wait(); err != nil {
log.Warningf("Ouputs returned error : %s", err)
reterr = err
done := make(chan error, 1)
go func() {
done <- outputsTomb.Wait()
}()
// wait for outputs to finish, max 3 seconds
select {
case err := <-done:
if err != nil {
log.Warningf("Outputs returned error : %s", err)
reterr = err
}
log.Debugf("outputs are done")
case <-time.After(3 * time.Second):
// this can happen if outputs are stuck in a http retry loop
log.Warningf("Outputs didn't finish in time, some events may have not been flushed")
}
log.Debugf("outputs are done")
// He's dead, Jim.
crowdsecTomb.Kill(nil)

7
plugins/notifications/http/Makefile → cmd/notification-dummy/Makefile
View file

@ -4,14 +4,13 @@ ifeq ($(OS), Windows_NT)
EXT = .exe
endif
PLUGIN=http
BINARY_NAME = notification-$(PLUGIN)$(EXT)
GO = go
GOBUILD = $(GO) build
BINARY_NAME = notification-dummy$(EXT)
build: clean
$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
.PHONY: clean
clean:

0
plugins/notifications/dummy/dummy.yaml → cmd/notification-dummy/dummy.yaml
View file

0
plugins/notifications/dummy/main.go → cmd/notification-dummy/main.go
View file

7
plugins/notifications/slack/Makefile → cmd/notification-email/Makefile
View file

@ -4,14 +4,13 @@ ifeq ($(OS), Windows_NT)
EXT = .exe
endif
PLUGIN=slack
BINARY_NAME = notification-$(PLUGIN)$(EXT)
GO = go
GOBUILD = $(GO) build
BINARY_NAME = notification-email$(EXT)
build: clean
$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
.PHONY: clean
clean:

14
plugins/notifications/email/email.yaml → cmd/notification-email/email.yaml
View file

@ -15,12 +15,14 @@ timeout: 20s # Time to wait for response from the plugin before conside
# The following template receives a list of models.Alert objects
# The output goes in the email message body
format: |
<html><body>
{{range . -}}
{{$alert := . -}}
{{range .Decisions -}}
<html><body><p><a href=https://www.whois.com/whois/{{.Value}}>{{.Value}}</a> will get <b>{{.Type}}</b> for next <b>{{.Duration}}</b> for triggering <b>{{.Scenario}}</b> on machine <b>{{$alert.MachineID}}</b>.</p> <p><a href=https://app.crowdsec.net/cti/{{.Value}}>CrowdSec CTI</a></p></body></html>
<p><a href="https://www.whois.com/whois/{{.Value}}">{{.Value}}</a> will get <b>{{.Type}}</b> for next <b>{{.Duration}}</b> for triggering <b>{{.Scenario}}</b> on machine <b>{{$alert.MachineID}}</b>.</p> <p><a href="https://app.crowdsec.net/cti/{{.Value}}">CrowdSec CTI</a></p>
{{end -}}
{{end -}}
</body></html>
smtp_host: # example: smtp.gmail.com
smtp_username: # Replace with your actual username
@ -35,7 +37,15 @@ receiver_emails:
# - email2@gmail.com
# One of "ssltls", "starttls", "none"
encryption_type: ssltls
encryption_type: "ssltls"
# If you need to set the HELO hostname:
# helo_host: "localhost"
# If the email server is hitting the default timeouts (10 seconds), you can increase them here
#
# connect_timeout: 10s
# send_timeout: 10s
---

23
plugins/notifications/email/main.go → cmd/notification-email/main.go
View file

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"os"
"time"
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
"github.com/hashicorp/go-hclog"
@ -47,6 +48,8 @@ type PluginConfig struct {
EncryptionType string `yaml:"encryption_type"`
AuthType string `yaml:"auth_type"`
HeloHost string `yaml:"helo_host"`
ConnectTimeout string `yaml:"connect_timeout"`
SendTimeout string `yaml:"send_timeout"`
}
type EmailPlugin struct {
@ -77,7 +80,7 @@ func (n *EmailPlugin) Configure(ctx context.Context, config *protobufs.Config) (
}
if d.ReceiverEmails == nil || len(d.ReceiverEmails) == 0 {
return nil, fmt.Errorf("Receiver emails are not set")
return nil, fmt.Errorf("receiver emails are not set")
}
n.ConfigByName[d.Name] = d
@ -108,6 +111,24 @@ func (n *EmailPlugin) Notify(ctx context.Context, notification *protobufs.Notifi
server.Authentication = AuthStringToType[cfg.AuthType]
server.Helo = cfg.HeloHost
var err error
if cfg.ConnectTimeout != "" {
server.ConnectTimeout, err = time.ParseDuration(cfg.ConnectTimeout)
if err != nil {
logger.Warn(fmt.Sprintf("invalid connect timeout '%s', using default '10s'", cfg.ConnectTimeout))
server.ConnectTimeout = 10 * time.Second
}
}
if cfg.SendTimeout != "" {
server.SendTimeout, err = time.ParseDuration(cfg.SendTimeout)
if err != nil {
logger.Warn(fmt.Sprintf("invalid send timeout '%s', using default '10s'", cfg.SendTimeout))
server.SendTimeout = 10 * time.Second
}
}
logger.Debug("making smtp connection")
smtpClient, err := server.Connect()
if err != nil {

7
plugins/notifications/splunk/Makefile → cmd/notification-http/Makefile
View file

@ -4,14 +4,13 @@ ifeq ($(OS), Windows_NT)
EXT = .exe
endif
PLUGIN=splunk
BINARY_NAME = notification-$(PLUGIN)$(EXT)
GO = go
GOBUILD = $(GO) build
BINARY_NAME = notification-http$(EXT)
build: clean
$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
.PHONY: clean
clean:

0
plugins/notifications/http/http.yaml → cmd/notification-http/http.yaml
View file

2
plugins/notifications/http/main.go → cmd/notification-http/main.go
View file

@ -63,7 +63,7 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific
logger.Debug(fmt.Sprintf("adding header %s: %s", headerName, headerValue))
request.Header.Add(headerName, headerValue)
}
logger.Debug(fmt.Sprintf("making HTTP %s call to %s with body %s", cfg.Method, cfg.URL, string(notification.Text)))
logger.Debug(fmt.Sprintf("making HTTP %s call to %s with body %s", cfg.Method, cfg.URL, notification.Text))
resp, err := client.Do(request)
if err != nil {
logger.Error(fmt.Sprintf("Failed to make HTTP request : %s", err))

4
plugins/notifications/sentinel/Makefile → cmd/notification-sentinel/Makefile
View file

@ -4,14 +4,14 @@ ifeq ($(OS), Windows_NT)
EXT = .exe
endif
# Go parameters
GO = go
GOBUILD = $(GO) build
BINARY_NAME = notification-sentinel$(EXT)
build: clean
$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
.PHONY: clean
clean:
@$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR)

4
plugins/notifications/sentinel/main.go → cmd/notification-sentinel/main.go
View file

@ -78,7 +78,7 @@ func (s *SentinelPlugin) Notify(ctx context.Context, notification *protobufs.Not
return &protobufs.Empty{}, err
}
req, err := http.NewRequest("POST", url, body)
req, err := http.NewRequest(http.MethodPost, url, body)
if err != nil {
logger.Error("failed to create request", "error", err)
return &protobufs.Empty{}, err
@ -98,7 +98,7 @@ func (s *SentinelPlugin) Notify(ctx context.Context, notification *protobufs.Not
defer resp.Body.Close()
logger.Debug("sent notification to sentinel", "status", resp.Status)
if resp.StatusCode != 200 {
if resp.StatusCode != http.StatusOK {
return &protobufs.Empty{}, fmt.Errorf("failed to send notification to sentinel: %s", resp.Status)
}

0
plugins/notifications/sentinel/sentinel.yaml → cmd/notification-sentinel/sentinel.yaml
View file

7
plugins/notifications/dummy/Makefile → cmd/notification-slack/Makefile
View file

@ -4,14 +4,13 @@ ifeq ($(OS), Windows_NT)
EXT = .exe
endif
PLUGIN = dummy
BINARY_NAME = notification-$(PLUGIN)$(EXT)
GO = go
GOBUILD = $(GO) build
BINARY_NAME = notification-slack$(EXT)
build: clean
$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
.PHONY: clean
clean:

0
plugins/notifications/slack/main.go → cmd/notification-slack/main.go
View file

0
plugins/notifications/slack/slack.yaml → cmd/notification-slack/slack.yaml
View file

17
cmd/notification-splunk/Makefile Normal file
View file

@ -0,0 +1,17 @@
ifeq ($(OS), Windows_NT)
SHELL := pwsh.exe
.SHELLFLAGS := -NoProfile -Command
EXT = .exe
endif
GO = go
GOBUILD = $(GO) build
BINARY_NAME = notification-splunk$(EXT)
build: clean
$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
.PHONY: clean
clean:
@$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR)

4
plugins/notifications/splunk/main.go → cmd/notification-splunk/main.go
View file

@ -58,7 +58,7 @@ func (s *Splunk) Notify(ctx context.Context, notification *protobufs.Notificatio
return &protobufs.Empty{}, err
}
req, err := http.NewRequest("POST", cfg.URL, strings.NewReader(string(data)))
req, err := http.NewRequest(http.MethodPost, cfg.URL, strings.NewReader(string(data)))
if err != nil {
return &protobufs.Empty{}, err
}
@ -70,7 +70,7 @@ func (s *Splunk) Notify(ctx context.Context, notification *protobufs.Notificatio
return &protobufs.Empty{}, err
}
if resp.StatusCode != 200 {
if resp.StatusCode != http.StatusOK {
content, err := io.ReadAll(resp.Body)
if err != nil {
return &protobufs.Empty{}, fmt.Errorf("got non 200 response and failed to read error %s", err)

0
plugins/notifications/splunk/splunk.yaml → cmd/notification-splunk/splunk.yaml
View file

10
debian/install vendored
View file

@ -6,8 +6,8 @@ config/patterns/* etc/crowdsec/patterns
config/crowdsec.service lib/systemd/system
# Referenced configs:
plugins/notifications/slack/slack.yaml etc/crowdsec/notifications/
plugins/notifications/http/http.yaml etc/crowdsec/notifications/
plugins/notifications/splunk/splunk.yaml etc/crowdsec/notifications/
plugins/notifications/email/email.yaml etc/crowdsec/notifications/
plugins/notifications/sentinel/sentinel.yaml etc/crowdsec/notifications/
cmd/notification-slack/slack.yaml etc/crowdsec/notifications/
cmd/notification-http/http.yaml etc/crowdsec/notifications/
cmd/notification-splunk/splunk.yaml etc/crowdsec/notifications/
cmd/notification-email/email.yaml etc/crowdsec/notifications/
cmd/notification-sentinel/sentinel.yaml etc/crowdsec/notifications/

10
debian/rules vendored
View file

@ -25,11 +25,11 @@ override_dh_auto_install:
mkdir -p debian/crowdsec/usr/lib/crowdsec/plugins/
mkdir -p debian/crowdsec/etc/crowdsec/notifications/
install -m 551 plugins/notifications/slack/notification-slack debian/crowdsec/usr/lib/crowdsec/plugins/
install -m 551 plugins/notifications/http/notification-http debian/crowdsec/usr/lib/crowdsec/plugins/
install -m 551 plugins/notifications/splunk/notification-splunk debian/crowdsec/usr/lib/crowdsec/plugins/
install -m 551 plugins/notifications/email/notification-email debian/crowdsec/usr/lib/crowdsec/plugins/
install -m 551 plugins/notifications/sentinel/notification-sentinel debian/crowdsec/usr/lib/crowdsec/plugins/
install -m 551 cmd/notification-slack/notification-slack debian/crowdsec/usr/lib/crowdsec/plugins/
install -m 551 cmd/notification-http/notification-http debian/crowdsec/usr/lib/crowdsec/plugins/
install -m 551 cmd/notification-splunk/notification-splunk debian/crowdsec/usr/lib/crowdsec/plugins/
install -m 551 cmd/notification-email/notification-email debian/crowdsec/usr/lib/crowdsec/plugins/
install -m 551 cmd/notification-sentinel/notification-sentinel debian/crowdsec/usr/lib/crowdsec/plugins/
cp cmd/crowdsec/crowdsec debian/crowdsec/usr/bin
cp cmd/crowdsec-cli/cscli debian/crowdsec/usr/bin

14
docker/README.md
View file

@ -19,11 +19,7 @@ All the following images are available on Docker Hub for the architectures
- `crowdsecurity/crowdsec:{version}`
Recommended for production usage. Also available on GitHub (ghcr.io).
- `crowdsecurity/crowdsec:dev`
The latest stable release.
Latest stable release recommended for production usage. Also available on GitHub (ghcr.io).
- `crowdsecurity/crowdsec:dev`
@ -190,6 +186,14 @@ It is not recommended anymore to bind-mount the full config.yaml file and you sh
If you want to use the [notification system](https://docs.crowdsec.net/docs/notification_plugins/intro), you have to use the full image (not slim) and mount at least a custom `profiles.yaml` and a notification configuration to `/etc/crowdsec/notifications`
```shell
docker run -d \
-v ./profiles.yaml:/etc/crowdsec/profiles.yaml \
-v ./http_notification.yaml:/etc/crowdsec/notifications/http_notification.yaml \
-p 8080:8080 -p 6060:6060 \
--name crowdsec crowdsecurity/crowdsec
```
# Deployment use cases
Crowdsec is composed of an `agent` that parses logs and creates `alerts`, and a

2
docker/test/Pipfile
View file

@ -1,7 +1,7 @@
[packages]
pytest-dotenv = "0.5.2"
pytest-xdist = "3.3.1"
pytest-cs = {ref = "0.7.16", git = "https://github.com/crowdsecurity/pytest-cs.git"}
pytest-cs = {ref = "0.7.18", git = "https://github.com/crowdsecurity/pytest-cs.git"}
[dev-packages]
gnureadline = "8.1.2"

122
docker/test/Pipfile.lock generated
View file

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "78f693678e411b7bdb5dd0280b7d6f8d9880069b331d44d96d32ba697275e30d"
"sha256": "64085783c9fec3a9eda976b7700b5bad7abd2b7a0f0670fa2209c52f3647be7f"
},
"pipfile-spec": 6,
"requires": {
@ -176,32 +176,32 @@
},
"cryptography": {
"hashes": [
"sha256:01f1d9e537f9a15b037d5d9ee442b8c22e3ae11ce65ea1f3316a41c78756b711",
"sha256:079347de771f9282fbfe0e0236c716686950c19dee1b76240ab09ce1624d76d7",
"sha256:182be4171f9332b6741ee818ec27daff9fb00349f706629f5cbf417bd50e66fd",
"sha256:192255f539d7a89f2102d07d7375b1e0a81f7478925b3bc2e0549ebf739dae0e",
"sha256:2a034bf7d9ca894720f2ec1d8b7b5832d7e363571828037f9e0c4f18c1b58a58",
"sha256:342f3767e25876751e14f8459ad85e77e660537ca0a066e10e75df9c9e9099f0",
"sha256:439c3cc4c0d42fa999b83ded80a9a1fb54d53c58d6e59234cfe97f241e6c781d",
"sha256:49c3222bb8f8e800aead2e376cbef687bc9e3cb9b58b29a261210456a7783d83",
"sha256:674b669d5daa64206c38e507808aae49904c988fa0a71c935e7006a3e1e83831",
"sha256:7a9a3bced53b7f09da251685224d6a260c3cb291768f54954e28f03ef14e3766",
"sha256:7af244b012711a26196450d34f483357e42aeddb04128885d95a69bd8b14b69b",
"sha256:7d230bf856164de164ecb615ccc14c7fc6de6906ddd5b491f3af90d3514c925c",
"sha256:84609ade00a6ec59a89729e87a503c6e36af98ddcd566d5f3be52e29ba993182",
"sha256:9a6673c1828db6270b76b22cc696f40cde9043eb90373da5c2f8f2158957f42f",
"sha256:9b6d717393dbae53d4e52684ef4f022444fc1cce3c48c38cb74fca29e1f08eaa",
"sha256:9c3fe6534d59d071ee82081ca3d71eed3210f76ebd0361798c74abc2bcf347d4",
"sha256:a719399b99377b218dac6cf547b6ec54e6ef20207b6165126a280b0ce97e0d2a",
"sha256:b332cba64d99a70c1e0836902720887fb4529ea49ea7f5462cf6640e095e11d2",
"sha256:d124682c7a23c9764e54ca9ab5b308b14b18eba02722b8659fb238546de83a76",
"sha256:d73f419a56d74fef257955f51b18d046f3506270a5fd2ac5febbfa259d6c0fa5",
"sha256:f0dc40e6f7aa37af01aba07277d3d64d5a03dc66d682097541ec4da03cc140ee",
"sha256:f14ad275364c8b4e525d018f6716537ae7b6d369c094805cae45300847e0894f",
"sha256:f772610fe364372de33d76edcd313636a25684edb94cee53fd790195f5989d14"
"sha256:004b6ccc95943f6a9ad3142cfabcc769d7ee38a3f60fb0dddbfb431f818c3a67",
"sha256:047c4603aeb4bbd8db2756e38f5b8bd7e94318c047cfe4efeb5d715e08b49311",
"sha256:0d9409894f495d465fe6fda92cb70e8323e9648af912d5b9141d616df40a87b8",
"sha256:23a25c09dfd0d9f28da2352503b23e086f8e78096b9fd585d1d14eca01613e13",
"sha256:2ed09183922d66c4ec5fdaa59b4d14e105c084dd0febd27452de8f6f74704143",
"sha256:35c00f637cd0b9d5b6c6bd11b6c3359194a8eba9c46d4e875a3660e3b400005f",
"sha256:37480760ae08065437e6573d14be973112c9e6dcaf5f11d00147ee74f37a3829",
"sha256:3b224890962a2d7b57cf5eeb16ccaafba6083f7b811829f00476309bce2fe0fd",
"sha256:5a0f09cefded00e648a127048119f77bc2b2ec61e736660b5789e638f43cc397",
"sha256:5b72205a360f3b6176485a333256b9bcd48700fc755fef51c8e7e67c4b63e3ac",
"sha256:7e53db173370dea832190870e975a1e09c86a879b613948f09eb49324218c14d",
"sha256:7febc3094125fc126a7f6fb1f420d0da639f3f32cb15c8ff0dc3997c4549f51a",
"sha256:80907d3faa55dc5434a16579952ac6da800935cd98d14dbd62f6f042c7f5e839",
"sha256:86defa8d248c3fa029da68ce61fe735432b047e32179883bdb1e79ed9bb8195e",
"sha256:8ac4f9ead4bbd0bc8ab2d318f97d85147167a488be0e08814a37eb2f439d5cf6",
"sha256:93530900d14c37a46ce3d6c9e6fd35dbe5f5601bf6b3a5c325c7bffc030344d9",
"sha256:9eeb77214afae972a00dee47382d2591abe77bdae166bda672fb1e24702a3860",
"sha256:b5f4dfe950ff0479f1f00eda09c18798d4f49b98f4e2006d644b3301682ebdca",
"sha256:c3391bd8e6de35f6f1140e50aaeb3e2b3d6a9012536ca23ab0d9c35ec18c8a91",
"sha256:c880eba5175f4307129784eca96f4e70b88e57aa3f680aeba3bab0e980b0f37d",
"sha256:cecfefa17042941f94ab54f769c8ce0fe14beff2694e9ac684176a2535bf9714",
"sha256:e40211b4923ba5a6dc9769eab704bdb3fbb58d56c5b336d30996c24fcf12aadb",
"sha256:efc8ad4e6fc4f1752ebfb58aefece8b4e3c4cae940b0994d43649bdfce8d0d4f"
],
"markers": "python_version >= '3.7'",
"version": "==41.0.2"
"version": "==41.0.4"
},
"docker": {
"hashes": [
@ -245,11 +245,11 @@
},
"pluggy": {
"hashes": [
"sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849",
"sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"
"sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12",
"sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"
],
"markers": "python_version >= '3.7'",
"version": "==1.2.0"
"markers": "python_version >= '3.8'",
"version": "==1.3.0"
},
"psutil": {
"hashes": [
@ -280,15 +280,15 @@
},
"pytest": {
"hashes": [
"sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32",
"sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"
"sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002",
"sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069"
],
"markers": "python_version >= '3.7'",
"version": "==7.4.0"
"version": "==7.4.2"
},
"pytest-cs": {
"git": "https://github.com/crowdsecurity/pytest-cs.git",
"ref": "4a3451084215053af8a48ff37507b4f86bf75c10"
"ref": "df835beabc539be7f7f627b21caa0d6ad333daae"
},
"pytest-datadir": {
"hashes": [
@ -324,7 +324,9 @@
},
"pyyaml": {
"hashes": [
"sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5",
"sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc",
"sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df",
"sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741",
"sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206",
"sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27",
@ -332,7 +334,10 @@
"sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62",
"sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98",
"sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696",
"sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290",
"sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9",
"sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d",
"sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6",
"sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867",
"sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47",
"sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486",
@ -340,9 +345,12 @@
"sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3",
"sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007",
"sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938",
"sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0",
"sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c",
"sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735",
"sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d",
"sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28",
"sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4",
"sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba",
"sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8",
"sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5",
@ -357,7 +365,9 @@
"sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43",
"sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859",
"sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673",
"sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54",
"sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a",
"sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b",
"sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab",
"sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa",
"sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c",
@ -386,28 +396,28 @@
},
"urllib3": {
"hashes": [
"sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11",
"sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4"
"sha256:13abf37382ea2ce6fb744d4dad67838eec857c9f4f57009891805e0b5e123594",
"sha256:ef16afa8ba34a1f989db38e1dbbe0c302e4289a47856990d0682e374563ce35e"
],
"markers": "python_version >= '3.7'",
"version": "==2.0.4"
"version": "==2.0.5"
},
"websocket-client": {
"hashes": [
"sha256:c951af98631d24f8df89ab1019fc365f2227c0892f12fd150e935607c79dd0dd",
"sha256:f1f9f2ad5291f0225a49efad77abf9e700b6fef553900623060dad6e26503b9d"
"sha256:3aad25d31284266bcfcfd1fd8a743f63282305a364b8d0948a43bd606acc652f",
"sha256:6cfc30d051ebabb73a5fa246efdcc14c8fbebbd0330f8984ac3bb6d9edd2ad03"
],
"markers": "python_version >= '3.7'",
"version": "==1.6.1"
"markers": "python_version >= '3.8'",
"version": "==1.6.3"
}
},
"develop": {
"asttokens": {
"hashes": [
"sha256:4622110b2a6f30b77e1473affaa97e711bc2f07d3f10848420ff1898edbe94f3",
"sha256:6b0ac9e93fb0335014d382b8fa9b3afa7df546984258005da0b9e7095b3deb1c"
"sha256:2e0171b991b2c959acc6c49318049236844a5da1d65ba2672c4880c1c894834e",
"sha256:cf8fc9e61a86461aa9fb161a14a0841a03c405fa829ac6b202670b3495d2ce69"
],
"version": "==2.2.1"
"version": "==2.4.0"
},
"backcall": {
"hashes": [
@ -474,19 +484,19 @@
},
"ipython": {
"hashes": [
"sha256:1d197b907b6ba441b692c48cf2a3a2de280dc0ac91a3405b39349a50272ca0a1",
"sha256:248aca623f5c99a6635bc3857677b7320b9b8039f99f070ee0d20a5ca5a8e6bf"
"sha256:2baeb5be6949eeebf532150f81746f8333e2ccce02de1c7eedde3f23ed5e9f1e",
"sha256:45a2c3a529296870a97b7de34eda4a31bee16bc7bf954e07d39abe49caf8f887"
],
"markers": "python_version >= '3.11'",
"version": "==8.14.0"
"version": "==8.15.0"
},
"jedi": {
"hashes": [
"sha256:203c1fd9d969ab8f2119ec0a3342e0b49910045abe6af0a3ae83a5764d54639e",
"sha256:bae794c30d07f6d910d32a7048af09b5a39ed740918da923c6b780790ebac612"
"sha256:bcf9894f1753969cbac8022a8c2eaee06bfa3724e4192470aaffe7eb6272b0c4",
"sha256:cb8ce23fbccff0025e9386b5cf85e892f94c9b822378f8da49970471335ac64e"
],
"markers": "python_version >= '3.6'",
"version": "==0.18.2"
"version": "==0.19.0"
},
"matplotlib-inline": {
"hashes": [
@ -543,11 +553,11 @@
},
"pygments": {
"hashes": [
"sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c",
"sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"
"sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692",
"sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"
],
"markers": "python_version >= '3.7'",
"version": "==2.15.1"
"version": "==2.16.1"
},
"six": {
"hashes": [
@ -566,11 +576,11 @@
},
"traitlets": {
"hashes": [
"sha256:9e6ec080259b9a5940c797d58b613b5e31441c2257b87c2e795c5228ae80d2d8",
"sha256:f6cde21a9c68cf756af02035f72d5a723bf607e862e7be33ece505abf4a3bad9"
"sha256:417745a96681fbb358e723d5346a547521f36e9bd0d50ba7ab368fff5d67aa54",
"sha256:f584ea209240466e66e91f3c81aa7d004ba4cf794990b0c775938a1544217cd1"
],
"markers": "python_version >= '3.7'",
"version": "==5.9.0"
"markers": "python_version >= '3.8'",
"version": "==5.10.0"
},
"wcwidth": {
"hashes": [

21
docker/test/tests/test_tls.py
View file

@ -4,7 +4,7 @@
Test agent-lapi and cscli-lapi communication via TLS, on the same container.
"""
import random
import uuid
from pytest_cs import Status
@ -140,7 +140,7 @@ def test_tls_lapi_var(crowdsec, flavor, certs_dir):
def test_tls_split_lapi_agent(crowdsec, flavor, certs_dir):
"""Server-only certificate, split containers"""
rand = random.randint(0, 10000)
rand = uuid.uuid1()
lapiname = 'lapi-' + str(rand)
agentname = 'agent-' + str(rand)
@ -193,7 +193,7 @@ def test_tls_split_lapi_agent(crowdsec, flavor, certs_dir):
def test_tls_mutual_split_lapi_agent(crowdsec, flavor, certs_dir):
"""Server and client certificates, split containers"""
rand = random.randint(0, 10000)
rand = uuid.uuid1()
lapiname = 'lapi-' + str(rand)
agentname = 'agent-' + str(rand)
@ -244,7 +244,7 @@ def test_tls_mutual_split_lapi_agent(crowdsec, flavor, certs_dir):
def test_tls_client_ou(crowdsec, certs_dir):
"""Check behavior of client certificate vs AGENTS_ALLOWED_OU"""
rand = random.randint(0, 10000)
rand = uuid.uuid1()
lapiname = 'lapi-' + str(rand)
agentname = 'agent-' + str(rand)
@ -287,6 +287,19 @@ def test_tls_client_ou(crowdsec, certs_dir):
lapi_env['AGENTS_ALLOWED_OU'] = 'custom-client-ou'
# change container names to avoid conflict
# recreate certificates because they need the new hostname
rand = uuid.uuid1()
lapiname = 'lapi-' + str(rand)
agentname = 'agent-' + str(rand)
agent_env['LOCAL_API_URL'] = f'https://{lapiname}:8080'
volumes = {
certs_dir(lapi_hostname=lapiname, agent_ou='custom-client-ou'): {'bind': '/etc/ssl/crowdsec', 'mode': 'ro'},
}
cs_lapi = crowdsec(name=lapiname, environment=lapi_env, volumes=volumes)
cs_agent = crowdsec(name=agentname, environment=agent_env, volumes=volumes)

18
go.mod
View file

@ -1,9 +1,13 @@
module github.com/crowdsecurity/crowdsec
go 1.20
go 1.21
// Don't use the toolchain directive to avoid uncontrolled downloads during
// a build, especially in sandboxed environments (freebsd, gentoo...).
// toolchain go1.21.3
require (
entgo.io/ent v0.11.3
entgo.io/ent v0.12.4
github.com/AlecAivazis/survey/v2 v2.2.7
github.com/Masterminds/semver/v3 v3.1.1
github.com/Masterminds/sprig/v3 v3.2.2
@ -21,7 +25,7 @@ require (
github.com/c-robinson/iplib v1.0.3
github.com/cespare/xxhash/v2 v2.2.0
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26
github.com/crowdsecurity/go-cs-lib v0.0.3
github.com/crowdsecurity/go-cs-lib v0.0.4
github.com/crowdsecurity/grokky v0.2.1
github.com/crowdsecurity/machineid v1.0.2
github.com/davecgh/go-spew v1.1.1
@ -68,12 +72,14 @@ require (
github.com/segmentio/kafka-go v0.4.34
github.com/shirou/gopsutil/v3 v3.23.5
github.com/sirupsen/logrus v1.9.3
github.com/slack-go/slack v0.12.2
github.com/spf13/cobra v1.7.0
github.com/stretchr/testify v1.8.4
github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
github.com/wasilibs/go-re2 v1.3.0
golang.org/x/crypto v0.10.0
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
github.com/xhit/go-simple-mail/v2 v2.16.0
golang.org/x/mod v0.11.0
golang.org/x/sys v0.10.0
google.golang.org/grpc v1.56.1
@ -90,7 +96,7 @@ require (
)
require (
ariga.io/atlas v0.7.2-0.20220927111110-867ee0cca56a // indirect
ariga.io/atlas v0.14.1-0.20230918065911-83ad451a4935 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/ahmetalpbalkan/dlog v0.0.0-20170105205344-4fb5f8204f26 // indirect
@ -126,6 +132,7 @@ require (
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/hashicorp/hcl/v2 v2.13.0 // indirect
github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect
github.com/huandu/xstrings v1.3.2 // indirect
@ -184,6 +191,7 @@ require (
github.com/tidwall/pretty v1.2.1 // indirect
github.com/tklauser/go-sysconf v0.3.11 // indirect
github.com/tklauser/numcpus v0.6.0 // indirect
github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.11 // indirect
github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
@ -196,7 +204,7 @@ require (
golang.org/x/term v0.10.0 // indirect
golang.org/x/text v0.11.0 // indirect
golang.org/x/time v0.2.0 // indirect
golang.org/x/tools v0.7.0 // indirect
golang.org/x/tools v0.8.1-0.20230428195545-5283a0178901 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect

44
go.sum
View file

@ -1,6 +1,7 @@
ariga.io/atlas v0.7.2-0.20220927111110-867ee0cca56a h1:6/nt4DODfgxzHTTg3tYy7YkVzruGQGZ/kRvXpA45KUo=
ariga.io/atlas v0.7.2-0.20220927111110-867ee0cca56a/go.mod h1:ft47uSh5hWGDCmQC9DsztZg6Xk+KagM5Ts/mZYKb9JE=
ariga.io/atlas v0.14.1-0.20230918065911-83ad451a4935 h1:JnYs/y8RJ3+MiIUp+3RgyyeO48VHLAZimqiaZYnMKk8=
ariga.io/atlas v0.14.1-0.20230918065911-83ad451a4935/go.mod h1:isZrlzJ5cpoCoKFoY9knZug7Lq4pP1cm8g3XciLZ0Pw=
bitbucket.org/creachadair/stringset v0.0.9 h1:L4vld9nzPt90UZNrXjNelTshD74ps4P5NGs3Iq6yN3o=
bitbucket.org/creachadair/stringset v0.0.9/go.mod h1:t+4WcQ4+PXTa8aQdNKe40ZP6iwesoMFWAxPGd3UGjyY=
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
@ -34,14 +35,16 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
entgo.io/ent v0.11.3 h1:F5FBGAWiDCGder7YT+lqMnyzXl6d0xU3xMBM/SO3CMc=
entgo.io/ent v0.11.3/go.mod h1:mvDhvynOzAsOe7anH7ynPPtMjA/eeXP96kAfweevyxc=
entgo.io/ent v0.12.4 h1:LddPnAyxls/O7DTXZvUGDj0NZIdGSu317+aoNLJWbD8=
entgo.io/ent v0.12.4/go.mod h1:Y3JVAjtlIk8xVZYSn3t3mf8xlZIn5SAOXZQxD6kKI+Q=
github.com/AlecAivazis/survey/v2 v2.2.7 h1:5NbxkF4RSKmpywYdcRgUmos1o+roJY8duCLZXbVjoig=
github.com/AlecAivazis/survey/v2 v2.2.7/go.mod h1:9DYvHgXtiXm6nCn+jXnOXLKbH+Yo9u8fAS/SduGdoPk=
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
@ -138,8 +141,8 @@ github.com/crowdsecurity/coraza/v3 v3.0.0-20230727080316-2348f4b3045f h1:7MgSs0r
github.com/crowdsecurity/coraza/v3 v3.0.0-20230727080316-2348f4b3045f/go.mod h1:YwM+m6iBdUn6P1eQKu+F+83bzkP0AzSEBCcVL//zh9c=
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
github.com/crowdsecurity/go-cs-lib v0.0.3 h1:NPSHTLS83A3wFyzV5R9Py8fBbTrVHu/1PQeaD7id4+I=
github.com/crowdsecurity/go-cs-lib v0.0.3/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
github.com/crowdsecurity/go-cs-lib v0.0.4 h1:mH3iqz8H8iH9YpldqCdojyKHy9z3JDhas/k6I8M0ims=
github.com/crowdsecurity/go-cs-lib v0.0.4/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4=
github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc=
@ -288,6 +291,7 @@ github.com/go-openapi/validate v0.20.0 h1:pzutNCCBZGZlE+u8HD3JZyWdc/TVbtVwlWUp8/
github.com/go-openapi/validate v0.20.0/go.mod h1:b60iJT+xNNLfaQJUqLI7946tYiFEOuE9E4k54HpKcJ0=
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
@ -302,7 +306,8 @@ github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfC
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho=
github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
@ -385,6 +390,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
@ -412,6 +418,9 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e h1:XmA6L9IPRdUr28a+SK/oMchGgQy159wvzXA5tJ7l+40=
github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e/go.mod h1:AFIo+02s+12CEg8Gzz9kzhCbmbq6JcKNrhHffCGA9z4=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c=
github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
github.com/hashicorp/go-plugin v1.4.10 h1:xUbmA4jC6Dq163/fWcp8P3JuHilrHHMLNRxzGQJ9hNk=
@ -488,6 +497,7 @@ github.com/jackc/puddle v1.2.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv
github.com/jarcoal/httpmock v1.1.0 h1:F47ChZj1Y2zFsCXxNkBPwNNKnAyOATcdQibk0qEdVCE=
github.com/jarcoal/httpmock v1.1.0/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE=
github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
@ -529,6 +539,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.4/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
@ -538,14 +549,15 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8=
github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw=
github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY=
github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
@ -693,6 +705,7 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
@ -719,6 +732,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/slack-go/slack v0.12.2 h1:x3OppyMyGIbbiyFhsBmpf9pwkUzMhthJMRNmNlA4LaQ=
github.com/slack-go/slack v0.12.2/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw=
github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
@ -764,6 +779,8 @@ github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+Kd
github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms=
github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4=
github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 h1:PM5hJF7HVfNWmCjMdEfbuOBNXSVF2cMFGgQTPdKCbwM=
github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208/go.mod h1:BzWtXXrXzZUvMacR0oF/fbDDgUPO8L36tDMmRAf14ns=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
@ -774,6 +791,7 @@ github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 h1:UFHFmFfixpmf
github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26/go.mod h1:IGhd0qMDsUa9acVjsbsT7bu3ktadtGOHI79+idTew/M=
github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
github.com/vjeantet/grok v1.0.1 h1:2rhIR7J4gThTgcZ1m2JY4TrJZNgjn985U28kT2wQrJ4=
github.com/vjeantet/grok v1.0.1/go.mod h1:ax1aAchzC6/QMXMcyzHQGZWaW1l195+uMYIkCWPCNIo=
github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI=
github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
@ -781,6 +799,7 @@ github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgq
github.com/wasilibs/go-re2 v1.3.0 h1:LFhBNzoStM3wMie6rN2slD1cuYH2CGiHpvNL3UtcsMw=
github.com/wasilibs/go-re2 v1.3.0/go.mod h1:AafrCXVvGRJJOImMajgJ2M7rVmWyisVK7sFshbxnVrg=
github.com/wasilibs/nottinygc v0.4.0 h1:h1TJMihMC4neN6Zq+WKpLxgd9xCFMw7O9ETLwY2exJQ=
github.com/wasilibs/nottinygc v0.4.0/go.mod h1:oDcIotskuYNMpqMF23l7Z8uzD4TC0WXHK8jetlB3HIo=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
@ -790,6 +809,8 @@ github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49
github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4=
github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
github.com/xhit/go-simple-mail/v2 v2.16.0 h1:ouGy/Ww4kuaqu2E2UrDw7SvLaziWTB60ICLkIkNVccA=
github.com/xhit/go-simple-mail/v2 v2.16.0/go.mod h1:b7P5ygho6SYE+VIqpxA6QkYfv4teeyG4MKqB3utRu98=
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@ -858,8 +879,6 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -1090,8 +1109,8 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc
golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4=
golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
golang.org/x/tools v0.8.1-0.20230428195545-5283a0178901 h1:0wxTF6pSjIIhNt7mo9GvjDfzyCOiWhmICgtO/Ah948s=
golang.org/x/tools v0.8.1-0.20230428195545-5283a0178901/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -1241,3 +1260,4 @@ sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h6
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

2
pkg/acquisition/modules/cloudwatch/cloudwatch.go
View file

@ -370,7 +370,7 @@ func (cw *CloudwatchSource) LogStreamManager(in chan LogStreamTailConfig, outCha
}
if cw.Config.StreamRegexp != nil {
match, err := regexp.Match(*cw.Config.StreamRegexp, []byte(newStream.StreamName))
match, err := regexp.MatchString(*cw.Config.StreamRegexp, newStream.StreamName)
if err != nil {
cw.logger.Warningf("invalid regexp : %s", err)
} else if !match {

4
pkg/acquisition/modules/docker/docker.go
View file

@ -392,14 +392,14 @@ func (d *DockerSource) EvalContainer(container dockerTypes.Container) (*Containe
}
for _, cont := range d.compiledContainerID {
if matched := cont.Match([]byte(container.ID)); matched {
if matched := cont.MatchString(container.ID); matched {
return &ContainerConfig{ID: container.ID, Name: container.Names[0], Labels: d.Config.Labels, Tty: d.getContainerTTY(container.ID)}, true
}
}
for _, cont := range d.compiledContainerName {
for _, name := range container.Names {
if matched := cont.Match([]byte(name)); matched {
if matched := cont.MatchString(name); matched {
return &ContainerConfig{ID: container.ID, Name: name, Labels: d.Config.Labels, Tty: d.getContainerTTY(container.ID)}, true
}
}

2
pkg/acquisition/modules/docker/docker_test.go
View file

@ -193,7 +193,7 @@ container_name_regexp:
actualLines++
ticker.Reset(1 * time.Second)
case <-ticker.C:
log.Infof("no more line to read")
log.Infof("no more lines to read")
dockerSource.t.Kill(nil)
return nil
}

6
pkg/acquisition/modules/file/file_test.go
View file

@ -410,9 +410,7 @@ force_inotify: true`, testPattern),
if tc.expectedLines != 0 {
fd, err := os.Create("test_files/stream.log")
if err != nil {
t.Fatalf("could not create test file : %s", err)
}
require.NoError(t, err, "could not create test file")
for i := 0; i < 5; i++ {
_, err = fmt.Fprintf(fd, "%d\n", i)
@ -424,7 +422,7 @@ force_inotify: true`, testPattern),
fd.Close()
// we sleep to make sure we detect the new file
time.Sleep(1 * time.Second)
time.Sleep(3 * time.Second)
os.Remove("test_files/stream.log")
assert.Equal(t, tc.expectedLines, actualLines)
}

3
pkg/acquisition/modules/kafka/kafka.go
View file

@ -149,7 +149,9 @@ func (k *KafkaSource) ReadMessage(out chan types.Event) error {
return nil
}
k.logger.Errorln(fmt.Errorf("while reading %s message: %w", dataSourceName, err))
continue
}
k.logger.Tracef("got message: %s", string(m.Value))
l := types.Line{
Raw: string(m.Value),
Labels: k.Config.Labels,
@ -223,7 +225,6 @@ func (kc *KafkaConfiguration) NewTLSConfig() (*tls.Config, error) {
caCertPool.AppendCertsFromPEM(caCert)
tlsConfig.RootCAs = caCertPool
tlsConfig.BuildNameToCertificate()
return &tlsConfig, err
}

2
pkg/alertcontext/alertcontext.go
View file

@ -3,12 +3,12 @@ package alertcontext
import (
"encoding/json"
"fmt"
"slices"
"strconv"
"github.com/antonmedv/expr"
"github.com/antonmedv/expr/vm"
log "github.com/sirupsen/logrus"
"golang.org/x/exp/slices"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/models"

19
pkg/apiclient/auth.go
View file

@ -96,10 +96,16 @@ func (r retryRoundTripper) ShouldRetry(statusCode int) bool {
func (r retryRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
var resp *http.Response
var err error
backoff := 0
for i := 0; i < r.maxAttempts; i++ {
maxAttempts := r.maxAttempts
if fflag.DisableHttpRetryBackoff.IsEnabled() {
maxAttempts = 1
}
for i := 0; i < maxAttempts; i++ {
if i > 0 {
if r.withBackOff && !fflag.DisableHttpRetryBackoff.IsEnabled() {
if r.withBackOff {
backoff += 10 + rand.Intn(20)
}
log.Infof("retrying in %d seconds (attempt %d of %d)", backoff, i+1, r.maxAttempts)
@ -115,7 +121,10 @@ func (r retryRoundTripper) RoundTrip(req *http.Request) (*http.Response, error)
clonedReq := cloneRequest(req)
resp, err = r.next.RoundTrip(clonedReq)
if err != nil {
log.Errorf("error while performing request: %s; %d retries left", err, r.maxAttempts-i-1)
left := maxAttempts - i - 1
if left > 0 {
log.Errorf("error while performing request: %s; %d retries left", err, left)
}
continue
}
if !r.ShouldRetry(resp.StatusCode) {
@ -264,7 +273,9 @@ func (t *JWTTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return resp, fmt.Errorf("performing jwt auth: %w", err)
}
log.Debugf("resp-jwt: %d", resp.StatusCode)
if resp != nil {
log.Debugf("resp-jwt: %d", resp.StatusCode)
}
return resp, nil
}

182
pkg/apiserver/apic.go
View file

@ -7,6 +7,7 @@ import (
"net"
"net/http"
"net/url"
"slices"
"strconv"
"strings"
"sync"
@ -15,7 +16,6 @@ import (
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"golang.org/x/exp/slices"
"gopkg.in/tomb.v2"
"github.com/crowdsecurity/go-cs-lib/ptr"
@ -43,8 +43,6 @@ const (
metricsIntervalDelta = time.Minute * 15
)
var SCOPE_CAPI_ALIAS_ALIAS string = "crowdsecurity/community-blocklist" //we don't use "CAPI" directly, to make it less confusing for the user
type apic struct {
// when changing the intervals in tests, always set *First too
// or they can be negative
@ -620,59 +618,57 @@ func (a *apic) PullTop(forcePull bool) error {
return nil
}
// if decisions is whitelisted: return representation of the whitelist ip or cidr
// if not whitelisted: empty string
func (a *apic) whitelistedBy(decision *models.Decision) string {
if decision.Value == nil {
return ""
}
ipval := net.ParseIP(*decision.Value)
for _, cidr := range a.whitelists.Cidrs {
if cidr.Contains(ipval) {
return cidr.String()
}
}
for _, ip := range a.whitelists.Ips {
if ip != nil && ip.Equal(ipval) {
return ip.String()
}
}
return ""
}
func (a *apic) ApplyApicWhitelists(decisions []*models.Decision) []*models.Decision {
if a.whitelists == nil {
if a.whitelists == nil || len(a.whitelists.Cidrs) == 0 && len(a.whitelists.Ips) == 0 {
return decisions
}
//deal with CAPI whitelists for fire. We want to avoid having a second list, so we shrink in place
outIdx := 0
for _, decision := range decisions {
if decision.Value == nil {
whitelister := a.whitelistedBy(decision)
if whitelister != "" {
log.Infof("%s from %s is whitelisted by %s", *decision.Value, *decision.Scenario, whitelister)
continue
}
skip := false
ipval := net.ParseIP(*decision.Value)
for _, cidr := range a.whitelists.Cidrs {
if skip {
break
}
if cidr.Contains(ipval) {
log.Infof("%s from %s is whitelisted by %s", *decision.Value, *decision.Scenario, cidr.String())
skip = true
}
}
for _, ip := range a.whitelists.Ips {
if skip {
break
}
if ip != nil && ip.Equal(ipval) {
log.Infof("%s from %s is whitelisted by %s", *decision.Value, *decision.Scenario, ip.String())
skip = true
}
}
if !skip {
decisions[outIdx] = decision
outIdx++
}
decisions[outIdx] = decision
outIdx++
}
//shrink the list, those are deleted items
decisions = decisions[:outIdx]
return decisions
return decisions[:outIdx]
}
func (a *apic) SaveAlerts(alertsFromCapi []*models.Alert, add_counters map[string]map[string]int, delete_counters map[string]map[string]int) error {
for idx, alert := range alertsFromCapi {
alertsFromCapi[idx] = setAlertScenario(add_counters, delete_counters, alert)
log.Debugf("%s has %d decisions", *alertsFromCapi[idx].Source.Scope, len(alertsFromCapi[idx].Decisions))
for _, alert := range alertsFromCapi {
setAlertScenario(alert, add_counters, delete_counters)
log.Debugf("%s has %d decisions", *alert.Source.Scope, len(alert.Decisions))
if a.dbClient.Type == "sqlite" && (a.dbClient.WalMode == nil || !*a.dbClient.WalMode) {
log.Warningf("sqlite is not using WAL mode, LAPI might become unresponsive when inserting the community blocklist")
}
alertID, inserted, deleted, err := a.dbClient.UpdateCommunityBlocklist(alertsFromCapi[idx])
alertID, inserted, deleted, err := a.dbClient.UpdateCommunityBlocklist(alert)
if err != nil {
return fmt.Errorf("while saving alert from %s: %w", *alertsFromCapi[idx].Source.Scope, err)
return fmt.Errorf("while saving alert from %s: %w", *alert.Source.Scope, err)
}
log.Printf("%s : added %d entries, deleted %d entries (alert:%d)", *alertsFromCapi[idx].Source.Scope, inserted, deleted, alertID)
log.Printf("%s : added %d entries, deleted %d entries (alert:%d)", *alert.Source.Scope, inserted, deleted, alertID)
}
return nil
@ -708,6 +704,60 @@ func (a *apic) ShouldForcePullBlocklist(blocklist *modelscapi.BlocklistLink) (bo
return false, nil
}
func (a *apic) updateBlocklist(client *apiclient.ApiClient, blocklist *modelscapi.BlocklistLink, add_counters map[string]map[string]int) error {
if blocklist.Scope == nil {
log.Warningf("blocklist has no scope")
return nil
}
if blocklist.Duration == nil {
log.Warningf("blocklist has no duration")
return nil
}
forcePull, err := a.ShouldForcePullBlocklist(blocklist)
if err != nil {
return fmt.Errorf("while checking if we should force pull blocklist %s: %w", *blocklist.Name, err)
}
blocklistConfigItemName := fmt.Sprintf("blocklist:%s:last_pull", *blocklist.Name)
var lastPullTimestamp *string
if !forcePull {
lastPullTimestamp, err = a.dbClient.GetConfigItem(blocklistConfigItemName)
if err != nil {
return fmt.Errorf("while getting last pull timestamp for blocklist %s: %w", *blocklist.Name, err)
}
}
decisions, hasChanged, err := client.Decisions.GetDecisionsFromBlocklist(context.Background(), blocklist, lastPullTimestamp)
if err != nil {
return fmt.Errorf("while getting decisions from blocklist %s: %w", *blocklist.Name, err)
}
if !hasChanged {
if lastPullTimestamp == nil {
log.Infof("blocklist %s hasn't been modified or there was an error reading it, skipping", *blocklist.Name)
} else {
log.Infof("blocklist %s hasn't been modified since %s, skipping", *blocklist.Name, *lastPullTimestamp)
}
return nil
}
err = a.dbClient.SetConfigItem(blocklistConfigItemName, time.Now().UTC().Format(http.TimeFormat))
if err != nil {
return fmt.Errorf("while setting last pull timestamp for blocklist %s: %w", *blocklist.Name, err)
}
if len(decisions) == 0 {
log.Infof("blocklist %s has no decisions", *blocklist.Name)
return nil
}
//apply APIC specific whitelists
decisions = a.ApplyApicWhitelists(decisions)
alert := createAlertForDecision(decisions[0])
alertsFromCapi := []*models.Alert{alert}
alertsFromCapi = fillAlertsWithDecisions(alertsFromCapi, decisions, add_counters)
err = a.SaveAlerts(alertsFromCapi, add_counters, nil)
if err != nil {
return fmt.Errorf("while saving alert from blocklist %s: %w", *blocklist.Name, err)
}
return nil
}
func (a *apic) UpdateBlocklists(links *modelscapi.GetDecisionsStreamResponseLinks, add_counters map[string]map[string]int) error {
if links == nil {
return nil
@ -722,69 +772,21 @@ func (a *apic) UpdateBlocklists(links *modelscapi.GetDecisionsStreamResponseLink
return fmt.Errorf("while creating default client: %w", err)
}
for _, blocklist := range links.Blocklists {
if blocklist.Scope == nil {
log.Warningf("blocklist has no scope")
continue
}
if blocklist.Duration == nil {
log.Warningf("blocklist has no duration")
continue
}
forcePull, err := a.ShouldForcePullBlocklist(blocklist)
if err != nil {
return fmt.Errorf("while checking if we should force pull blocklist %s: %w", *blocklist.Name, err)
}
blocklistConfigItemName := fmt.Sprintf("blocklist:%s:last_pull", *blocklist.Name)
var lastPullTimestamp *string
if !forcePull {
lastPullTimestamp, err = a.dbClient.GetConfigItem(blocklistConfigItemName)
if err != nil {
return fmt.Errorf("while getting last pull timestamp for blocklist %s: %w", *blocklist.Name, err)
}
}
decisions, has_changed, err := defaultClient.Decisions.GetDecisionsFromBlocklist(context.Background(), blocklist, lastPullTimestamp)
if err != nil {
return fmt.Errorf("while getting decisions from blocklist %s: %w", *blocklist.Name, err)
}
if !has_changed {
if lastPullTimestamp == nil {
log.Infof("blocklist %s hasn't been modified or there was an error reading it, skipping", *blocklist.Name)
} else {
log.Infof("blocklist %s hasn't been modified since %s, skipping", *blocklist.Name, *lastPullTimestamp)
}
continue
}
err = a.dbClient.SetConfigItem(blocklistConfigItemName, time.Now().UTC().Format(http.TimeFormat))
if err != nil {
return fmt.Errorf("while setting last pull timestamp for blocklist %s: %w", *blocklist.Name, err)
}
if len(decisions) == 0 {
log.Infof("blocklist %s has no decisions", *blocklist.Name)
continue
}
//apply APIC specific whitelists
decisions = a.ApplyApicWhitelists(decisions)
alert := createAlertForDecision(decisions[0])
alertsFromCapi := []*models.Alert{alert}
alertsFromCapi = fillAlertsWithDecisions(alertsFromCapi, decisions, add_counters)
err = a.SaveAlerts(alertsFromCapi, add_counters, nil)
if err != nil {
return fmt.Errorf("while saving alert from blocklist %s: %w", *blocklist.Name, err)
if err := a.updateBlocklist(defaultClient, blocklist, add_counters); err != nil {
return err
}
}
return nil
}
func setAlertScenario(add_counters map[string]map[string]int, delete_counters map[string]map[string]int, alert *models.Alert) *models.Alert {
func setAlertScenario(alert *models.Alert, add_counters map[string]map[string]int, delete_counters map[string]map[string]int) {
if *alert.Source.Scope == types.CAPIOrigin {
*alert.Source.Scope = SCOPE_CAPI_ALIAS_ALIAS
*alert.Source.Scope = types.CommunityBlocklistPullSourceScope
alert.Scenario = ptr.Of(fmt.Sprintf("update : +%d/-%d IPs", add_counters[types.CAPIOrigin]["all"], delete_counters[types.CAPIOrigin]["all"]))
} else if *alert.Source.Scope == types.ListOrigin {
*alert.Source.Scope = fmt.Sprintf("%s:%s", types.ListOrigin, *alert.Scenario)
alert.Scenario = ptr.Of(fmt.Sprintf("update : +%d/-%d IPs", add_counters[types.ListOrigin][*alert.Scenario], delete_counters[types.ListOrigin][*alert.Scenario]))
}
return alert
}
func (a *apic) Pull() error {

17
pkg/apiserver/apic_metrics.go
View file

@ -2,10 +2,10 @@ package apiserver
import (
"context"
"slices"
"time"
log "github.com/sirupsen/logrus"
"golang.org/x/exp/slices"
"github.com/crowdsecurity/go-cs-lib/ptr"
"github.com/crowdsecurity/go-cs-lib/trace"
@ -83,7 +83,7 @@ func (a *apic) SendMetrics(stop chan (bool)) {
// intervals must always be > 0
metInts := []time.Duration{1*time.Millisecond, a.metricsIntervalFirst, a.metricsInterval}
log.Infof("Start send metrics to CrowdSec Central API (interval: %s once, then %s)",
log.Infof("Start sending metrics to CrowdSec Central API (interval: %s once, then %s)",
metInts[1].Round(time.Second), metInts[2])
count := -1
@ -129,12 +129,15 @@ func (a *apic) SendMetrics(stop chan (bool)) {
metTicker.Stop()
metrics, err := a.GetMetrics()
if err != nil {
log.Errorf("unable to get metrics (%s), will retry", err)
log.Errorf("unable to get metrics (%s)", err)
}
log.Info("capi metrics: sending")
_, _, err = a.apiClient.Metrics.Add(context.Background(), metrics)
if err != nil {
log.Errorf("capi metrics: failed: %s", err)
// metrics are nil if they could not be retrieved
if metrics != nil {
log.Info("capi metrics: sending")
_, _, err = a.apiClient.Metrics.Add(context.Background(), metrics)
if err != nil {
log.Errorf("capi metrics: failed: %s", err)
}
}
metTicker.Reset(nextMetInt())
case <-a.metricsTomb.Dying(): // if one apic routine is dying, do we kill the others?

4
pkg/apiserver/apic_test.go
View file

@ -689,7 +689,7 @@ func TestAPICWhitelists(t *testing.T) {
alertScenario[alert.SourceScope]++
}
assert.Equal(t, 3, len(alertScenario))
assert.Equal(t, 1, alertScenario[SCOPE_CAPI_ALIAS_ALIAS])
assert.Equal(t, 1, alertScenario[types.CommunityBlocklistPullSourceScope])
assert.Equal(t, 1, alertScenario["lists:blocklist1"])
assert.Equal(t, 1, alertScenario["lists:blocklist2"])
@ -818,7 +818,7 @@ func TestAPICPullTop(t *testing.T) {
alertScenario[alert.SourceScope]++
}
assert.Equal(t, 3, len(alertScenario))
assert.Equal(t, 1, alertScenario[SCOPE_CAPI_ALIAS_ALIAS])
assert.Equal(t, 1, alertScenario[types.CommunityBlocklistPullSourceScope])
assert.Equal(t, 1, alertScenario["lists:blocklist1"])
assert.Equal(t, 1, alertScenario["lists:blocklist2"])

2
pkg/apiserver/jwt_test.go
View file

@ -55,7 +55,7 @@ func TestLogin(t *testing.T) {
router.ServeHTTP(w, req)
assert.Equal(t, 401, w.Code)
assert.Equal(t, "{\"code\":401,\"message\":\"input format error\"}", w.Body.String())
assert.Equal(t, "{\"code\":401,\"message\":\"validation failure list:\\npassword in body is required\"}", w.Body.String())
//Validate machine
err = ValidateMachine("test", config.API.Server.DbConfig)

4
pkg/apiserver/middlewares/v1/api_key.go
View file

@ -33,7 +33,9 @@ func GenerateAPIKey(n int) (string, error) {
if _, err := rand.Read(bytes); err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(bytes), nil
encoded := base64.StdEncoding.EncodeToString(bytes)
// the '=' can cause issues on some bouncers
return strings.TrimRight(encoded, "="), nil
}
func NewAPIKey(dbClient *database.Client) *APIKey {

264
pkg/apiserver/middlewares/v1/jwt.go
View file

@ -2,6 +2,7 @@ package v1
import (
"crypto/rand"
"errors"
"fmt"
"net/http"
"os"
@ -16,7 +17,6 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/gin-gonic/gin"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"golang.org/x/crypto/bcrypt"
)
@ -46,142 +46,176 @@ func IdentityHandler(c *gin.Context) interface{} {
}
}
func (j *JWT) Authenticator(c *gin.Context) (interface{}, error) {
var loginInput models.WatcherAuthRequest
var scenarios string
var err error
var scenariosInput []string
var clientMachine *ent.Machine
var machineID string
if c.Request.TLS != nil && len(c.Request.TLS.PeerCertificates) > 0 {
if j.TlsAuth == nil {
c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
c.Abort()
return nil, errors.New("TLS auth is not configured")
}
validCert, extractedCN, err := j.TlsAuth.ValidateCert(c)
if err != nil {
log.Error(err)
c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
c.Abort()
return nil, errors.Wrap(err, "while trying to validate client cert")
}
if !validCert {
c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
c.Abort()
return nil, fmt.Errorf("failed cert authentication")
}
machineID = fmt.Sprintf("%s@%s", extractedCN, c.ClientIP())
clientMachine, err = j.DbClient.Ent.Machine.Query().
Where(machine.MachineId(machineID)).
First(j.DbClient.CTX)
if ent.IsNotFound(err) {
//Machine was not found, let's create it
log.Printf("machine %s not found, create it", machineID)
//let's use an apikey as the password, doesn't matter in this case (generatePassword is only available in cscli)
pwd, err := GenerateAPIKey(dummyAPIKeySize)
if err != nil {
log.WithFields(log.Fields{
"ip": c.ClientIP(),
"cn": extractedCN,
}).Errorf("error generating password: %s", err)
return nil, fmt.Errorf("error generating password")
}
password := strfmt.Password(pwd)
clientMachine, err = j.DbClient.CreateMachine(&machineID, &password, "", true, true, types.TlsAuthType)
if err != nil {
return "", errors.Wrapf(err, "while creating machine entry for %s", machineID)
}
} else if err != nil {
return "", errors.Wrapf(err, "while selecting machine entry for %s", machineID)
} else {
if clientMachine.AuthType != types.TlsAuthType {
return "", errors.Errorf("machine %s attempted to auth with TLS cert but it is configured to use %s", machineID, clientMachine.AuthType)
}
machineID = clientMachine.MachineId
loginInput := struct {
Scenarios []string `json:"scenarios"`
}{
Scenarios: []string{},
}
err := c.ShouldBindJSON(&loginInput)
if err != nil {
return "", errors.Wrap(err, "missing scenarios list in login request for TLS auth")
}
scenariosInput = loginInput.Scenarios
}
type authInput struct {
machineID string
clientMachine *ent.Machine
scenariosInput []string
}
} else {
//normal auth
if err := c.ShouldBindJSON(&loginInput); err != nil {
return "", errors.Wrap(err, "missing")
}
if err := loginInput.Validate(strfmt.Default); err != nil {
return "", errors.New("input format error")
}
machineID = *loginInput.MachineID
password := *loginInput.Password
scenariosInput = loginInput.Scenarios
clientMachine, err = j.DbClient.Ent.Machine.Query().
Where(machine.MachineId(machineID)).
First(j.DbClient.CTX)
if err != nil {
log.Printf("Error machine login for %s : %+v ", machineID, err)
return nil, err
}
func (j *JWT) authTLS(c *gin.Context) (*authInput, error) {
ret := authInput{}
if clientMachine == nil {
log.Errorf("Nothing for '%s'", machineID)
return nil, jwt.ErrFailedAuthentication
}
if clientMachine.AuthType != types.PasswordAuthType {
return nil, errors.Errorf("machine %s attempted to auth with password but it is configured to use %s", machineID, clientMachine.AuthType)
}
if !clientMachine.IsValidated {
return nil, fmt.Errorf("machine %s not validated", machineID)
}
if err = bcrypt.CompareHashAndPassword([]byte(clientMachine.Password), []byte(password)); err != nil {
return nil, jwt.ErrFailedAuthentication
}
//end of normal auth
if j.TlsAuth == nil {
c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
c.Abort()
return nil, errors.New("TLS auth is not configured")
}
if len(scenariosInput) > 0 {
for _, scenario := range scenariosInput {
validCert, extractedCN, err := j.TlsAuth.ValidateCert(c)
if err != nil {
log.Error(err)
c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
c.Abort()
return nil, fmt.Errorf("while trying to validate client cert: %w", err)
}
if !validCert {
c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
c.Abort()
return nil, fmt.Errorf("failed cert authentication")
}
ret.machineID = fmt.Sprintf("%s@%s", extractedCN, c.ClientIP())
ret.clientMachine, err = j.DbClient.Ent.Machine.Query().
Where(machine.MachineId(ret.machineID)).
First(j.DbClient.CTX)
if ent.IsNotFound(err) {
//Machine was not found, let's create it
log.Infof("machine %s not found, create it", ret.machineID)
//let's use an apikey as the password, doesn't matter in this case (generatePassword is only available in cscli)
pwd, err := GenerateAPIKey(dummyAPIKeySize)
if err != nil {
log.WithFields(log.Fields{
"ip": c.ClientIP(),
"cn": extractedCN,
}).Errorf("error generating password: %s", err)
return nil, fmt.Errorf("error generating password")
}
password := strfmt.Password(pwd)
ret.clientMachine, err = j.DbClient.CreateMachine(&ret.machineID, &password, "", true, true, types.TlsAuthType)
if err != nil {
return nil, fmt.Errorf("while creating machine entry for %s: %w", ret.machineID, err)
}
} else if err != nil {
return nil, fmt.Errorf("while selecting machine entry for %s: %w", ret.machineID, err)
} else {
if ret.clientMachine.AuthType != types.TlsAuthType {
return nil, fmt.Errorf("machine %s attempted to auth with TLS cert but it is configured to use %s", ret.machineID, ret.clientMachine.AuthType)
}
ret.machineID = ret.clientMachine.MachineId
}
loginInput := struct {
Scenarios []string `json:"scenarios"`
}{
Scenarios: []string{},
}
err = c.ShouldBindJSON(&loginInput)
if err != nil {
return nil, fmt.Errorf("missing scenarios list in login request for TLS auth: %w", err)
}
ret.scenariosInput = loginInput.Scenarios
return &ret, nil
}
func (j *JWT) authPlain(c *gin.Context) (*authInput, error) {
var loginInput models.WatcherAuthRequest
var err error
ret := authInput{}
if err = c.ShouldBindJSON(&loginInput); err != nil {
return nil, fmt.Errorf("missing: %w", err)
}
if err = loginInput.Validate(strfmt.Default); err != nil {
return nil, err
}
ret.machineID = *loginInput.MachineID
password := *loginInput.Password
ret.scenariosInput = loginInput.Scenarios
ret.clientMachine, err = j.DbClient.Ent.Machine.Query().
Where(machine.MachineId(ret.machineID)).
First(j.DbClient.CTX)
if err != nil {
log.Infof("Error machine login for %s : %+v ", ret.machineID, err)
return nil, err
}
if ret.clientMachine == nil {
log.Errorf("Nothing for '%s'", ret.machineID)
return nil, jwt.ErrFailedAuthentication
}
if ret.clientMachine.AuthType != types.PasswordAuthType {
return nil, fmt.Errorf("machine %s attempted to auth with password but it is configured to use %s", ret.machineID, ret.clientMachine.AuthType)
}
if !ret.clientMachine.IsValidated {
return nil, fmt.Errorf("machine %s not validated", ret.machineID)
}
if err := bcrypt.CompareHashAndPassword([]byte(ret.clientMachine.Password), []byte(password)); err != nil {
return nil, jwt.ErrFailedAuthentication
}
return &ret, nil
}
func (j *JWT) Authenticator(c *gin.Context) (interface{}, error) {
var err error
var auth *authInput
if c.Request.TLS != nil && len(c.Request.TLS.PeerCertificates) > 0 {
auth, err = j.authTLS(c)
if err != nil {
return nil, err
}
} else {
auth, err = j.authPlain(c)
if err != nil {
return nil, err
}
}
var scenarios string
if len(auth.scenariosInput) > 0 {
for _, scenario := range auth.scenariosInput {
if scenarios == "" {
scenarios = scenario
} else {
scenarios += "," + scenario
}
}
err = j.DbClient.UpdateMachineScenarios(scenarios, clientMachine.ID)
err = j.DbClient.UpdateMachineScenarios(scenarios, auth.clientMachine.ID)
if err != nil {
log.Errorf("Failed to update scenarios list for '%s': %s\n", machineID, err)
log.Errorf("Failed to update scenarios list for '%s': %s\n", auth.machineID, err)
return nil, jwt.ErrFailedAuthentication
}
}
if clientMachine.IpAddress == "" {
err = j.DbClient.UpdateMachineIP(c.ClientIP(), clientMachine.ID)
if auth.clientMachine.IpAddress == "" {
err = j.DbClient.UpdateMachineIP(c.ClientIP(), auth.clientMachine.ID)
if err != nil {
log.Errorf("Failed to update ip address for '%s': %s\n", machineID, err)
log.Errorf("Failed to update ip address for '%s': %s\n", auth.machineID, err)
return nil, jwt.ErrFailedAuthentication
}
}
if clientMachine.IpAddress != c.ClientIP() && clientMachine.IpAddress != "" {
log.Warningf("new IP address detected for machine '%s': %s (old: %s)", clientMachine.MachineId, c.ClientIP(), clientMachine.IpAddress)
err = j.DbClient.UpdateMachineIP(c.ClientIP(), clientMachine.ID)
if auth.clientMachine.IpAddress != c.ClientIP() && auth.clientMachine.IpAddress != "" {
log.Warningf("new IP address detected for machine '%s': %s (old: %s)", auth.clientMachine.MachineId, c.ClientIP(), auth.clientMachine.IpAddress)
err = j.DbClient.UpdateMachineIP(c.ClientIP(), auth.clientMachine.ID)
if err != nil {
log.Errorf("Failed to update ip address for '%s': %s\n", clientMachine.MachineId, err)
log.Errorf("Failed to update ip address for '%s': %s\n", auth.clientMachine.MachineId, err)
return nil, jwt.ErrFailedAuthentication
}
}
@ -192,13 +226,13 @@ func (j *JWT) Authenticator(c *gin.Context) (interface{}, error) {
return nil, jwt.ErrFailedAuthentication
}
if err := j.DbClient.UpdateMachineVersion(useragent[1], clientMachine.ID); err != nil {
log.Errorf("unable to update machine '%s' version '%s': %s", clientMachine.MachineId, useragent[1], err)
if err := j.DbClient.UpdateMachineVersion(useragent[1], auth.clientMachine.ID); err != nil {
log.Errorf("unable to update machine '%s' version '%s': %s", auth.clientMachine.MachineId, useragent[1], err)
log.Errorf("bad user agent from : %s", c.ClientIP())
return nil, jwt.ErrFailedAuthentication
}
return &models.WatcherAuthRequest{
MachineID: &machineID,
MachineID: &auth.machineID,
}, nil
}

64
pkg/csconfig/api.go
View file

@ -3,7 +3,9 @@ package csconfig
import (
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"io"
"net"
"os"
"strings"
@ -56,7 +58,6 @@ type CTICfg struct {
}
func (a *CTICfg) Load() error {
if a.Key == nil {
*a.Enabled = false
}
@ -331,40 +332,59 @@ type capiWhitelists struct {
Cidrs []string `yaml:"cidrs"`
}
func parseCapiWhitelists(fd io.Reader) (*CapiWhitelist, error) {
fromCfg := capiWhitelists{}
decoder := yaml.NewDecoder(fd)
if err := decoder.Decode(&fromCfg); err != nil {
if errors.Is(err, io.EOF) {
return nil, fmt.Errorf("empty file")
}
return nil, err
}
ret := &CapiWhitelist{
Ips: make([]net.IP, len(fromCfg.Ips)),
Cidrs: make([]*net.IPNet, len(fromCfg.Cidrs)),
}
for idx, v := range fromCfg.Ips {
ip := net.ParseIP(v)
if ip == nil {
return nil, fmt.Errorf("invalid IP address: %s", v)
}
ret.Ips[idx] = ip
}
for idx, v := range fromCfg.Cidrs {
_, tnet, err := net.ParseCIDR(v)
if err != nil {
return nil, err
}
ret.Cidrs[idx] = tnet
}
return ret, nil
}
func (s *LocalApiServerCfg) LoadCapiWhitelists() error {
if s.CapiWhitelistsPath == "" {
return nil
}
if _, err := os.Stat(s.CapiWhitelistsPath); os.IsNotExist(err) {
return fmt.Errorf("capi whitelist file '%s' does not exist", s.CapiWhitelistsPath)
}
fd, err := os.Open(s.CapiWhitelistsPath)
if err != nil {
return fmt.Errorf("unable to open capi whitelist file '%s': %s", s.CapiWhitelistsPath, err)
return fmt.Errorf("while opening capi whitelist file: %s", err)
}
var fromCfg capiWhitelists
s.CapiWhitelists = &CapiWhitelist{}
defer fd.Close()
decoder := yaml.NewDecoder(fd)
if err := decoder.Decode(&fromCfg); err != nil {
return fmt.Errorf("while parsing capi whitelist file '%s': %s", s.CapiWhitelistsPath, err)
}
for _, v := range fromCfg.Ips {
ip := net.ParseIP(v)
if ip == nil {
return fmt.Errorf("unable to parse ip whitelist '%s'", v)
}
s.CapiWhitelists.Ips = append(s.CapiWhitelists.Ips, ip)
}
for _, v := range fromCfg.Cidrs {
_, tnet, err := net.ParseCIDR(v)
if err != nil {
return fmt.Errorf("unable to parse cidr whitelist '%s' : %v", v, err)
}
s.CapiWhitelists.Cidrs = append(s.CapiWhitelists.Cidrs, tnet)
s.CapiWhitelists, err = parseCapiWhitelists(fd)
if err != nil {
return fmt.Errorf("while parsing capi whitelist file '%s': %w", s.CapiWhitelistsPath, err)
}
return nil
}

155
pkg/csconfig/api_test.go
View file

@ -1,7 +1,7 @@
package csconfig
import (
"fmt"
"net"
"os"
"path/filepath"
"strings"
@ -9,6 +9,7 @@ import (
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/cstest"
@ -25,7 +26,7 @@ func TestLoadLocalApiClientCfg(t *testing.T) {
{
name: "basic valid configuration",
input: &LocalApiClientCfg{
CredentialsFilePath: "./tests/lapi-secrets.yaml",
CredentialsFilePath: "./testdata/lapi-secrets.yaml",
},
expected: &ApiCredentialsCfg{
URL: "http://localhost:8080/",
@ -36,7 +37,7 @@ func TestLoadLocalApiClientCfg(t *testing.T) {
{
name: "invalid configuration",
input: &LocalApiClientCfg{
CredentialsFilePath: "./tests/bad_lapi-secrets.yaml",
CredentialsFilePath: "./testdata/bad_lapi-secrets.yaml",
},
expected: &ApiCredentialsCfg{},
expectedErr: "field unknown_key not found in type csconfig.ApiCredentialsCfg",
@ -44,15 +45,15 @@ func TestLoadLocalApiClientCfg(t *testing.T) {
{
name: "invalid configuration filepath",
input: &LocalApiClientCfg{
CredentialsFilePath: "./tests/nonexist_lapi-secrets.yaml",
CredentialsFilePath: "./testdata/nonexist_lapi-secrets.yaml",
},
expected: nil,
expectedErr: "open ./tests/nonexist_lapi-secrets.yaml: " + cstest.FileNotFoundMessage,
expectedErr: "open ./testdata/nonexist_lapi-secrets.yaml: " + cstest.FileNotFoundMessage,
},
{
name: "valid configuration with insecure skip verify",
input: &LocalApiClientCfg{
CredentialsFilePath: "./tests/lapi-secrets.yaml",
CredentialsFilePath: "./testdata/lapi-secrets.yaml",
InsecureSkipVerify: ptr.Of(false),
},
expected: &ApiCredentialsCfg{
@ -87,7 +88,7 @@ func TestLoadOnlineApiClientCfg(t *testing.T) {
{
name: "basic valid configuration",
input: &OnlineApiClientCfg{
CredentialsFilePath: "./tests/online-api-secrets.yaml",
CredentialsFilePath: "./testdata/online-api-secrets.yaml",
},
expected: &ApiCredentialsCfg{
URL: "http://crowdsec.api",
@ -98,7 +99,7 @@ func TestLoadOnlineApiClientCfg(t *testing.T) {
{
name: "invalid configuration",
input: &OnlineApiClientCfg{
CredentialsFilePath: "./tests/bad_lapi-secrets.yaml",
CredentialsFilePath: "./testdata/bad_lapi-secrets.yaml",
},
expected: &ApiCredentialsCfg{},
expectedErr: "failed unmarshaling api server credentials",
@ -106,14 +107,14 @@ func TestLoadOnlineApiClientCfg(t *testing.T) {
{
name: "missing field configuration",
input: &OnlineApiClientCfg{
CredentialsFilePath: "./tests/bad_online-api-secrets.yaml",
CredentialsFilePath: "./testdata/bad_online-api-secrets.yaml",
},
expected: nil,
},
{
name: "invalid configuration filepath",
input: &OnlineApiClientCfg{
CredentialsFilePath: "./tests/nonexist_online-api-secrets.yaml",
CredentialsFilePath: "./testdata/nonexist_online-api-secrets.yaml",
},
expected: &ApiCredentialsCfg{},
expectedErr: "failed to read api server credentials",
@ -136,27 +137,23 @@ func TestLoadOnlineApiClientCfg(t *testing.T) {
func TestLoadAPIServer(t *testing.T) {
tmpLAPI := &LocalApiServerCfg{
ProfilesPath: "./tests/profiles.yaml",
}
if err := tmpLAPI.LoadProfiles(); err != nil {
t.Fatalf("loading tmp profiles: %+v", err)
ProfilesPath: "./testdata/profiles.yaml",
}
err := tmpLAPI.LoadProfiles()
require.NoError(t, err)
LogDirFullPath, err := filepath.Abs("./testdata")
require.NoError(t, err)
LogDirFullPath, err := filepath.Abs("./tests")
if err != nil {
t.Fatal(err)
}
logLevel := log.InfoLevel
config := &Config{}
fcontent, err := os.ReadFile("./tests/config.yaml")
if err != nil {
t.Fatal(err)
}
fcontent, err := os.ReadFile("./testdata/config.yaml")
require.NoError(t, err)
configData := os.ExpandEnv(string(fcontent))
err = yaml.UnmarshalStrict([]byte(configData), &config)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
tests := []struct {
name string
input *Config
@ -171,18 +168,18 @@ func TestLoadAPIServer(t *testing.T) {
Server: &LocalApiServerCfg{
ListenURI: "http://crowdsec.api",
OnlineClient: &OnlineApiClientCfg{
CredentialsFilePath: "./tests/online-api-secrets.yaml",
CredentialsFilePath: "./testdata/online-api-secrets.yaml",
},
ProfilesPath: "./tests/profiles.yaml",
ProfilesPath: "./testdata/profiles.yaml",
PapiLogLevel: &logLevel,
},
},
DbConfig: &DatabaseCfg{
Type: "sqlite",
DbPath: "./tests/test.db",
DbPath: "./testdata/test.db",
},
Common: &CommonCfg{
LogDir: "./tests/",
LogDir: "./testdata/",
LogMedia: "stdout",
},
DisableAPI: false,
@ -192,9 +189,10 @@ func TestLoadAPIServer(t *testing.T) {
ListenURI: "http://crowdsec.api",
TLS: nil,
DbConfig: &DatabaseCfg{
DbPath: "./tests/test.db",
Type: "sqlite",
MaxOpenConns: ptr.Of(DEFAULT_MAX_OPEN_CONNS),
DbPath: "./testdata/test.db",
Type: "sqlite",
MaxOpenConns: ptr.Of(DEFAULT_MAX_OPEN_CONNS),
DecisionBulkSize: defaultDecisionBulkSize,
},
ConsoleConfigPath: DefaultConfigPath("console.yaml"),
ConsoleConfig: &ConsoleConfig{
@ -207,7 +205,7 @@ func TestLoadAPIServer(t *testing.T) {
LogDir: LogDirFullPath,
LogMedia: "stdout",
OnlineClient: &OnlineApiClientCfg{
CredentialsFilePath: "./tests/online-api-secrets.yaml",
CredentialsFilePath: "./testdata/online-api-secrets.yaml",
Credentials: &ApiCredentialsCfg{
URL: "http://crowdsec.api",
Login: "test",
@ -215,7 +213,7 @@ func TestLoadAPIServer(t *testing.T) {
},
},
Profiles: tmpLAPI.Profiles,
ProfilesPath: "./tests/profiles.yaml",
ProfilesPath: "./testdata/profiles.yaml",
UseForwardedForHeaders: false,
PapiLogLevel: &logLevel,
},
@ -228,34 +226,91 @@ func TestLoadAPIServer(t *testing.T) {
Server: &LocalApiServerCfg{},
},
Common: &CommonCfg{
LogDir: "./tests/",
LogDir: "./testdata/",
LogMedia: "stdout",
},
DisableAPI: false,
},
expected: &LocalApiServerCfg{
Enable: ptr.Of(true),
Enable: ptr.Of(true),
PapiLogLevel: &logLevel,
},
expectedErr: "no database configuration provided",
},
}
for idx, test := range tests {
err := test.input.LoadAPIServer()
if err == nil && test.expectedErr != "" {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("Test number %d/%d expected error, didn't get it", idx+1, len(tests))
} else if test.expectedErr != "" {
fmt.Printf("ERR: %+v\n", err)
if !strings.HasPrefix(fmt.Sprintf("%s", err), test.expectedErr) {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
test.expectedErr,
fmt.Sprintf("%s", err))
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
err := tc.input.LoadAPIServer()
cstest.RequireErrorContains(t, err, tc.expectedErr)
if tc.expectedErr != "" {
return
}
assert.Equal(t, test.expected, test.input.API.Server)
}
assert.Equal(t, tc.expected, tc.input.API.Server)
})
}
}
func mustParseCIDRNet(t *testing.T, s string) *net.IPNet {
_, ipNet, err := net.ParseCIDR(s)
require.NoError(t, err)
return ipNet
}
func TestParseCapiWhitelists(t *testing.T) {
tests := []struct {
name string
input string
expected *CapiWhitelist
expectedErr string
}{
{
name: "empty file",
input: "",
expected: &CapiWhitelist{
Ips: []net.IP{},
Cidrs: []*net.IPNet{},
},
expectedErr: "empty file",
},
{
name: "empty ip and cidr",
input: `{"ips": [], "cidrs": []}`,
expected: &CapiWhitelist{
Ips: []net.IP{},
Cidrs: []*net.IPNet{},
},
},
{
name: "some ip",
input: `{"ips": ["1.2.3.4"]}`,
expected: &CapiWhitelist{
Ips: []net.IP{net.IPv4(1, 2, 3, 4)},
Cidrs: []*net.IPNet{},
},
},
{
name: "some cidr",
input: `{"cidrs": ["1.2.3.0/24"]}`,
expected: &CapiWhitelist{
Ips: []net.IP{},
Cidrs: []*net.IPNet{mustParseCIDRNet(t, "1.2.3.0/24")},
},
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
wl, err := parseCapiWhitelists(strings.NewReader(tc.input))
cstest.RequireErrorContains(t, err, tc.expectedErr)
if tc.expectedErr != "" {
return
}
assert.Equal(t, tc.expected, wl)
})
}
}

79
pkg/csconfig/common_test.go
View file

@ -1,44 +1,41 @@
package csconfig
import (
"fmt"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/cstest"
)
func TestLoadCommon(t *testing.T) {
pidDirPath := "./tests"
LogDirFullPath, err := filepath.Abs("./tests/log/")
if err != nil {
t.Fatal(err)
}
pidDirPath := "./testdata"
LogDirFullPath, err := filepath.Abs("./testdata/log/")
require.NoError(t, err)
WorkingDirFullPath, err := filepath.Abs("./tests")
if err != nil {
t.Fatal(err)
}
WorkingDirFullPath, err := filepath.Abs("./testdata")
require.NoError(t, err)
tests := []struct {
name string
Input *Config
expectedResult *CommonCfg
err string
name string
input *Config
expected *CommonCfg
expectedErr string
}{
{
name: "basic valid configuration",
Input: &Config{
input: &Config{
Common: &CommonCfg{
Daemonize: true,
PidDir: "./tests",
PidDir: "./testdata",
LogMedia: "file",
LogDir: "./tests/log/",
WorkingDir: "./tests/",
LogDir: "./testdata/log/",
WorkingDir: "./testdata/",
},
},
expectedResult: &CommonCfg{
expected: &CommonCfg{
Daemonize: true,
PidDir: pidDirPath,
LogMedia: "file",
@ -48,15 +45,15 @@ func TestLoadCommon(t *testing.T) {
},
{
name: "empty working dir",
Input: &Config{
input: &Config{
Common: &CommonCfg{
Daemonize: true,
PidDir: "./tests",
PidDir: "./testdata",
LogMedia: "file",
LogDir: "./tests/log/",
LogDir: "./testdata/log/",
},
},
expectedResult: &CommonCfg{
expected: &CommonCfg{
Daemonize: true,
PidDir: pidDirPath,
LogMedia: "file",
@ -64,31 +61,23 @@ func TestLoadCommon(t *testing.T) {
},
},
{
name: "no common",
Input: &Config{},
expectedResult: nil,
name: "no common",
input: &Config{},
expected: nil,
expectedErr: "no common block provided in configuration file",
},
}
for idx, test := range tests {
err := test.Input.LoadCommon()
if err == nil && test.err != "" {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
} else if test.err != "" {
if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
test.err,
fmt.Sprintf("%s", err))
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
err := tc.input.LoadCommon()
cstest.RequireErrorContains(t, err, tc.expectedErr)
if tc.expectedErr != "" {
return
}
}
isOk := assert.Equal(t, test.expectedResult, test.Input.Common)
if !isOk {
t.Fatalf("TEST '%s': NOK", test.name)
} else {
fmt.Printf("TEST '%s': OK\n", test.name)
}
assert.Equal(t, tc.expected, tc.input.Common)
})
}
}

11
pkg/csconfig/config.go
View file

@ -1,3 +1,5 @@
// Package csconfig contains the configuration structures for crowdsec and cscli.
package csconfig
import (
@ -37,15 +39,6 @@ type Config struct {
Hub *Hub `yaml:"-"`
}
func (c *Config) Dump() error {
out, err := yaml.Marshal(c)
if err != nil {
return fmt.Errorf("failed marshaling config: %w", err)
}
fmt.Printf("%s", string(out))
return nil
}
func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*Config, string, error) {
patcher := yamlpatch.NewPatcher(configFile, ".local")
patcher.SetQuiet(quiet)

25
pkg/csconfig/config_test.go
View file

@ -5,42 +5,43 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/cstest"
)
func TestNormalLoad(t *testing.T) {
_, _, err := NewConfig("./tests/config.yaml", false, false, false)
_, _, err := NewConfig("./testdata/config.yaml", false, false, false)
require.NoError(t, err)
_, _, err = NewConfig("./tests/xxx.yaml", false, false, false)
assert.EqualError(t, err, "while reading yaml file: open ./tests/xxx.yaml: "+cstest.FileNotFoundMessage)
_, _, err = NewConfig("./testdata/xxx.yaml", false, false, false)
assert.EqualError(t, err, "while reading yaml file: open ./testdata/xxx.yaml: "+cstest.FileNotFoundMessage)
_, _, err = NewConfig("./tests/simulation.yaml", false, false, false)
assert.EqualError(t, err, "./tests/simulation.yaml: yaml: unmarshal errors:\n line 1: field simulation not found in type csconfig.Config")
_, _, err = NewConfig("./testdata/simulation.yaml", false, false, false)
assert.EqualError(t, err, "./testdata/simulation.yaml: yaml: unmarshal errors:\n line 1: field simulation not found in type csconfig.Config")
}
func TestNewCrowdSecConfig(t *testing.T) {
tests := []struct {
name string
expectedResult *Config
name string
expected *Config
}{
{
name: "new configuration: basic",
expectedResult: &Config{},
name: "new configuration: basic",
expected: &Config{},
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
result := &Config{}
assert.Equal(t, tc.expectedResult, result)
assert.Equal(t, tc.expected, result)
})
}
}
func TestDefaultConfig(t *testing.T) {
x := NewDefaultConfig()
err := x.Dump()
require.NoError(t, err)
_, err := yaml.Marshal(x)
require.NoError(t, err, "failed marshaling config: %s", err)
}

20
pkg/csconfig/console.go
View file

@ -82,23 +82,3 @@ func (c *LocalApiServerCfg) LoadConsoleConfig() error {
return nil
}
func (c *LocalApiServerCfg) DumpConsoleConfig() error {
var out []byte
var err error
if out, err = yaml.Marshal(c.ConsoleConfig); err != nil {
return fmt.Errorf("while marshaling ConsoleConfig (for %s): %w", c.ConsoleConfigPath, err)
}
if c.ConsoleConfigPath == "" {
c.ConsoleConfigPath = DefaultConsoleConfigFilePath
log.Debugf("Empty console_path, defaulting to %s", c.ConsoleConfigPath)
}
if err := os.WriteFile(c.ConsoleConfigPath, out, 0600); err != nil {
return fmt.Errorf("while dumping console config to %s: %w", c.ConsoleConfigPath, err)
}
return nil
}

72
pkg/csconfig/crowdsec_service_test.go
View file

@ -1,24 +1,23 @@
package csconfig
import (
"fmt"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/cstest"
"github.com/crowdsecurity/go-cs-lib/ptr"
"github.com/stretchr/testify/require"
)
func TestLoadCrowdsec(t *testing.T) {
acquisFullPath, err := filepath.Abs("./tests/acquis.yaml")
acquisFullPath, err := filepath.Abs("./testdata/acquis.yaml")
require.NoError(t, err)
acquisInDirFullPath, err := filepath.Abs("./tests/acquis/acquis.yaml")
acquisInDirFullPath, err := filepath.Abs("./testdata/acquis/acquis.yaml")
require.NoError(t, err)
acquisDirFullPath, err := filepath.Abs("./tests/acquis")
acquisDirFullPath, err := filepath.Abs("./testdata/acquis")
require.NoError(t, err)
hubFullPath, err := filepath.Abs("./hub")
@ -27,42 +26,42 @@ func TestLoadCrowdsec(t *testing.T) {
dataFullPath, err := filepath.Abs("./data")
require.NoError(t, err)
configDirFullPath, err := filepath.Abs("./tests")
configDirFullPath, err := filepath.Abs("./testdata")
require.NoError(t, err)
hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
require.NoError(t, err)
contextFileFullPath, err := filepath.Abs("./tests/context.yaml")
contextFileFullPath, err := filepath.Abs("./testdata/context.yaml")
require.NoError(t, err)
tests := []struct {
name string
input *Config
expectedResult *CrowdsecServiceCfg
expectedErr string
name string
input *Config
expected *CrowdsecServiceCfg
expectedErr string
}{
{
name: "basic valid configuration",
input: &Config{
ConfigPaths: &ConfigurationPaths{
ConfigDir: "./tests",
ConfigDir: "./testdata",
DataDir: "./data",
HubDir: "./hub",
},
API: &APICfg{
Client: &LocalApiClientCfg{
CredentialsFilePath: "./tests/lapi-secrets.yaml",
CredentialsFilePath: "./testdata/lapi-secrets.yaml",
},
},
Crowdsec: &CrowdsecServiceCfg{
AcquisitionFilePath: "./tests/acquis.yaml",
SimulationFilePath: "./tests/simulation.yaml",
ConsoleContextPath: "./tests/context.yaml",
AcquisitionFilePath: "./testdata/acquis.yaml",
SimulationFilePath: "./testdata/simulation.yaml",
ConsoleContextPath: "./testdata/context.yaml",
ConsoleContextValueLength: 2500,
},
},
expectedResult: &CrowdsecServiceCfg{
expected: &CrowdsecServiceCfg{
Enable: ptr.Of(true),
AcquisitionDirPath: "",
ConsoleContextPath: contextFileFullPath,
@ -76,7 +75,7 @@ func TestLoadCrowdsec(t *testing.T) {
OutputRoutinesCount: 1,
ConsoleContextValueLength: 2500,
AcquisitionFiles: []string{acquisFullPath},
SimulationFilePath: "./tests/simulation.yaml",
SimulationFilePath: "./testdata/simulation.yaml",
ContextToSend: map[string][]string{
"source_ip": {"evt.Parsed.source_ip"},
},
@ -89,23 +88,23 @@ func TestLoadCrowdsec(t *testing.T) {
name: "basic valid configuration with acquisition dir",
input: &Config{
ConfigPaths: &ConfigurationPaths{
ConfigDir: "./tests",
ConfigDir: "./testdata",
DataDir: "./data",
HubDir: "./hub",
},
API: &APICfg{
Client: &LocalApiClientCfg{
CredentialsFilePath: "./tests/lapi-secrets.yaml",
CredentialsFilePath: "./testdata/lapi-secrets.yaml",
},
},
Crowdsec: &CrowdsecServiceCfg{
AcquisitionFilePath: "./tests/acquis.yaml",
AcquisitionDirPath: "./tests/acquis/",
SimulationFilePath: "./tests/simulation.yaml",
ConsoleContextPath: "./tests/context.yaml",
AcquisitionFilePath: "./testdata/acquis.yaml",
AcquisitionDirPath: "./testdata/acquis/",
SimulationFilePath: "./testdata/simulation.yaml",
ConsoleContextPath: "./testdata/context.yaml",
},
},
expectedResult: &CrowdsecServiceCfg{
expected: &CrowdsecServiceCfg{
Enable: ptr.Of(true),
AcquisitionDirPath: acquisDirFullPath,
AcquisitionFilePath: acquisFullPath,
@ -122,7 +121,7 @@ func TestLoadCrowdsec(t *testing.T) {
ContextToSend: map[string][]string{
"source_ip": {"evt.Parsed.source_ip"},
},
SimulationFilePath: "./tests/simulation.yaml",
SimulationFilePath: "./testdata/simulation.yaml",
SimulationConfig: &SimulationConfig{
Simulation: ptr.Of(false),
},
@ -132,13 +131,13 @@ func TestLoadCrowdsec(t *testing.T) {
name: "no acquisition file and dir",
input: &Config{
ConfigPaths: &ConfigurationPaths{
ConfigDir: "./tests",
ConfigDir: "./testdata",
DataDir: "./data",
HubDir: "./hub",
},
API: &APICfg{
Client: &LocalApiClientCfg{
CredentialsFilePath: "./tests/lapi-secrets.yaml",
CredentialsFilePath: "./testdata/lapi-secrets.yaml",
},
},
Crowdsec: &CrowdsecServiceCfg{
@ -146,7 +145,7 @@ func TestLoadCrowdsec(t *testing.T) {
ConsoleContextValueLength: 10,
},
},
expectedResult: &CrowdsecServiceCfg{
expected: &CrowdsecServiceCfg{
Enable: ptr.Of(true),
AcquisitionDirPath: "",
AcquisitionFilePath: "",
@ -173,18 +172,18 @@ func TestLoadCrowdsec(t *testing.T) {
name: "non existing acquisition file",
input: &Config{
ConfigPaths: &ConfigurationPaths{
ConfigDir: "./tests",
ConfigDir: "./testdata",
DataDir: "./data",
HubDir: "./hub",
},
API: &APICfg{
Client: &LocalApiClientCfg{
CredentialsFilePath: "./tests/lapi-secrets.yaml",
CredentialsFilePath: "./testdata/lapi-secrets.yaml",
},
},
Crowdsec: &CrowdsecServiceCfg{
ConsoleContextPath: "",
AcquisitionFilePath: "./tests/acquis_not_exist.yaml",
AcquisitionFilePath: "./testdata/acquis_not_exist.yaml",
},
},
expectedErr: cstest.FileNotFoundMessage,
@ -193,26 +192,25 @@ func TestLoadCrowdsec(t *testing.T) {
name: "agent disabled",
input: &Config{
ConfigPaths: &ConfigurationPaths{
ConfigDir: "./tests",
ConfigDir: "./testdata",
DataDir: "./data",
HubDir: "./hub",
},
},
expectedResult: nil,
expected: nil,
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
fmt.Printf("TEST '%s'\n", tc.name)
err := tc.input.LoadCrowdsec()
cstest.RequireErrorContains(t, err, tc.expectedErr)
if tc.expectedErr != "" {
return
}
require.Equal(t, tc.expectedResult, tc.input.Crowdsec)
require.Equal(t, tc.expected, tc.input.Crowdsec)
})
}
}

71
pkg/csconfig/cscli_test.go
View file

@ -1,52 +1,45 @@
package csconfig
import (
"fmt"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/cstest"
)
func TestLoadCSCLI(t *testing.T) {
hubFullPath, err := filepath.Abs("./hub")
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
dataFullPath, err := filepath.Abs("./data")
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
configDirFullPath, err := filepath.Abs("./tests")
if err != nil {
t.Fatal(err)
}
configDirFullPath, err := filepath.Abs("./testdata")
require.NoError(t, err)
hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
tests := []struct {
name string
Input *Config
expectedResult *CscliCfg
err string
name string
input *Config
expected *CscliCfg
expectedErr string
}{
{
name: "basic valid configuration",
Input: &Config{
input: &Config{
ConfigPaths: &ConfigurationPaths{
ConfigDir: "./tests",
ConfigDir: "./testdata",
DataDir: "./data",
HubDir: "./hub",
HubIndexFile: "./hub/.index.json",
},
},
expectedResult: &CscliCfg{
expected: &CscliCfg{
ConfigDir: configDirFullPath,
DataDir: dataFullPath,
HubDir: hubFullPath,
@ -54,31 +47,23 @@ func TestLoadCSCLI(t *testing.T) {
},
},
{
name: "no configuration path",
Input: &Config{},
expectedResult: &CscliCfg{},
name: "no configuration path",
input: &Config{},
expected: &CscliCfg{},
expectedErr: "no configuration paths provided",
},
}
for idx, test := range tests {
err := test.Input.LoadCSCLI()
if err == nil && test.err != "" {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
} else if test.err != "" {
if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
test.err,
fmt.Sprintf("%s", err))
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
err := tc.input.LoadCSCLI()
cstest.RequireErrorContains(t, err, tc.expectedErr)
if tc.expectedErr != "" {
return
}
}
isOk := assert.Equal(t, test.expectedResult, test.Input.Cscli)
if !isOk {
t.Fatalf("TEST '%s': NOK", test.name)
} else {
fmt.Printf("TEST '%s': OK\n", test.name)
}
assert.Equal(t, tc.expected, tc.input.Cscli)
})
}
}

44
pkg/csconfig/database.go
View file

@ -10,21 +10,28 @@ import (
"github.com/crowdsecurity/go-cs-lib/ptr"
)
var DEFAULT_MAX_OPEN_CONNS = 100
const (
DEFAULT_MAX_OPEN_CONNS = 100
defaultDecisionBulkSize = 1000
// we need an upper bound due to the sqlite limit of 32k variables in a query
// we have 15 variables per decision, so 32768/15 = 2184.5333
maxDecisionBulkSize = 2000
)
type DatabaseCfg struct {
User string `yaml:"user"`
Password string `yaml:"password"`
DbName string `yaml:"db_name"`
Sslmode string `yaml:"sslmode"`
Host string `yaml:"host"`
Port int `yaml:"port"`
DbPath string `yaml:"db_path"`
Type string `yaml:"type"`
Flush *FlushDBCfg `yaml:"flush"`
LogLevel *log.Level `yaml:"log_level"`
MaxOpenConns *int `yaml:"max_open_conns,omitempty"`
UseWal *bool `yaml:"use_wal,omitempty"`
User string `yaml:"user"`
Password string `yaml:"password"`
DbName string `yaml:"db_name"`
Sslmode string `yaml:"sslmode"`
Host string `yaml:"host"`
Port int `yaml:"port"`
DbPath string `yaml:"db_path"`
Type string `yaml:"type"`
Flush *FlushDBCfg `yaml:"flush"`
LogLevel *log.Level `yaml:"log_level"`
MaxOpenConns *int `yaml:"max_open_conns,omitempty"`
UseWal *bool `yaml:"use_wal,omitempty"`
DecisionBulkSize int `yaml:"decision_bulk_size,omitempty"`
}
type AuthGCCfg struct {
@ -60,11 +67,20 @@ func (c *Config) LoadDBConfig() error {
c.DbConfig.MaxOpenConns = ptr.Of(DEFAULT_MAX_OPEN_CONNS)
}
if c.DbConfig.DecisionBulkSize == 0 {
log.Tracef("No decision_bulk_size value provided, using default value of %d", defaultDecisionBulkSize)
c.DbConfig.DecisionBulkSize = defaultDecisionBulkSize
}
if c.DbConfig.DecisionBulkSize > maxDecisionBulkSize {
log.Warningf("decision_bulk_size too high (%d), setting to the maximum value of %d", c.DbConfig.DecisionBulkSize, maxDecisionBulkSize)
c.DbConfig.DecisionBulkSize = maxDecisionBulkSize
}
if c.DbConfig.Type == "sqlite" {
if c.DbConfig.UseWal == nil {
log.Warning("You are using sqlite without WAL, this can have a performance impact. If you do not store the database in a network share, set db_config.use_wal to true. Set explicitly to false to disable this warning.")
}
}
return nil

59
pkg/csconfig/database_test.go
View file

@ -1,28 +1,27 @@
package csconfig
import (
"fmt"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/crowdsecurity/go-cs-lib/cstest"
"github.com/crowdsecurity/go-cs-lib/ptr"
)
func TestLoadDBConfig(t *testing.T) {
tests := []struct {
name string
Input *Config
expectedResult *DatabaseCfg
err string
name string
input *Config
expected *DatabaseCfg
expectedErr string
}{
{
name: "basic valid configuration",
Input: &Config{
input: &Config{
DbConfig: &DatabaseCfg{
Type: "sqlite",
DbPath: "./tests/test.db",
DbPath: "./testdata/test.db",
MaxOpenConns: ptr.Of(10),
},
Cscli: &CscliCfg{},
@ -30,37 +29,31 @@ func TestLoadDBConfig(t *testing.T) {
Server: &LocalApiServerCfg{},
},
},
expectedResult: &DatabaseCfg{
Type: "sqlite",
DbPath: "./tests/test.db",
MaxOpenConns: ptr.Of(10),
expected: &DatabaseCfg{
Type: "sqlite",
DbPath: "./testdata/test.db",
MaxOpenConns: ptr.Of(10),
DecisionBulkSize: defaultDecisionBulkSize,
},
},
{
name: "no configuration path",
Input: &Config{},
expectedResult: nil,
name: "no configuration path",
input: &Config{},
expected: nil,
expectedErr: "no database configuration provided",
},
}
for idx, test := range tests {
err := test.Input.LoadDBConfig()
if err == nil && test.err != "" {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
} else if test.err != "" {
if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
test.err,
fmt.Sprintf("%s", err))
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
err := tc.input.LoadDBConfig()
cstest.RequireErrorContains(t, err, tc.expectedErr)
if tc.expectedErr != "" {
return
}
}
isOk := assert.Equal(t, test.expectedResult, test.Input.DbConfig)
if !isOk {
t.Fatalf("TEST '%s': NOK", test.name)
} else {
fmt.Printf("TEST '%s': OK\n", test.name)
}
assert.Equal(t, tc.expected, tc.input.DbConfig)
})
}
}

15
pkg/csconfig/fflag.go
View file

@ -10,7 +10,6 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/fflag"
)
// LoadFeatureFlagsEnv parses the environment variables to enable feature flags.
func LoadFeatureFlagsEnv(logger *log.Logger) error {
if err := fflag.Crowdsec.SetFromEnv(logger); err != nil {
@ -19,13 +18,18 @@ func LoadFeatureFlagsEnv(logger *log.Logger) error {
return nil
}
// LoadFeatureFlags parses feature.yaml to enable feature flags.
// FeatureFlagsFileLocation returns the path to the feature.yaml file.
// The file is in the same directory as config.yaml, which is provided
// as the fist parameter. This can be different than ConfigPaths.ConfigDir
func LoadFeatureFlagsFile(configPath string, logger *log.Logger) error {
// because we have not read config.yaml yet so we don't know the value of ConfigDir.
func GetFeatureFilePath(configPath string) string {
dir := filepath.Dir(configPath)
featurePath := filepath.Join(dir, "feature.yaml")
return filepath.Join(dir, "feature.yaml")
}
// LoadFeatureFlags parses feature.yaml to enable feature flags.
func LoadFeatureFlagsFile(configPath string, logger *log.Logger) error {
featurePath := GetFeatureFilePath(configPath)
if err := fflag.Crowdsec.SetFromYamlFile(featurePath, logger); err != nil {
return fmt.Errorf("file %s: %s", featurePath, err)
@ -33,7 +37,6 @@ func LoadFeatureFlagsFile(configPath string, logger *log.Logger) error {
return nil
}
// ListFeatureFlags returns a list of the enabled feature flags.
func ListFeatureFlags() string {
enabledFeatures := fflag.Crowdsec.GetEnabledFeatures()

16
pkg/csconfig/hub.go
View file

@ -2,10 +2,10 @@ package csconfig
/*cscli specific config, such as hub directory*/
type Hub struct {
HubDir string `yaml:"-"`
ConfigDir string `yaml:"-"`
HubIndexFile string `yaml:"-"`
DataDir string `yaml:"-"`
HubIndexFile string
HubDir string
InstallDir string
InstallDataDir string
}
func (c *Config) LoadHub() error {
@ -14,10 +14,10 @@ func (c *Config) LoadHub() error {
}
c.Hub = &Hub{
HubIndexFile: c.ConfigPaths.HubIndexFile,
ConfigDir: c.ConfigPaths.ConfigDir,
HubDir: c.ConfigPaths.HubDir,
DataDir: c.ConfigPaths.DataDir,
HubIndexFile: c.ConfigPaths.HubIndexFile,
HubDir: c.ConfigPaths.HubDir,
InstallDir: c.ConfigPaths.ConfigDir,
InstallDataDir: c.ConfigPaths.DataDir,
}
return nil

85
pkg/csconfig/hub_test.go
View file

@ -1,94 +1,79 @@
package csconfig
import (
"fmt"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/cstest"
)
func TestLoadHub(t *testing.T) {
hubFullPath, err := filepath.Abs("./hub")
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
dataFullPath, err := filepath.Abs("./data")
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
configDirFullPath, err := filepath.Abs("./tests")
if err != nil {
t.Fatal(err)
}
configDirFullPath, err := filepath.Abs("./testdata")
require.NoError(t, err)
hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
tests := []struct {
name string
Input *Config
expectedResult *Hub
err string
name string
input *Config
expected *Hub
expectedErr string
}{
{
name: "basic valid configuration",
Input: &Config{
input: &Config{
ConfigPaths: &ConfigurationPaths{
ConfigDir: "./tests",
ConfigDir: "./testdata",
DataDir: "./data",
HubDir: "./hub",
HubIndexFile: "./hub/.index.json",
},
},
expectedResult: &Hub{
ConfigDir: configDirFullPath,
DataDir: dataFullPath,
HubDir: hubFullPath,
HubIndexFile: hubIndexFileFullPath,
expected: &Hub{
HubDir: hubFullPath,
HubIndexFile: hubIndexFileFullPath,
InstallDir: configDirFullPath,
InstallDataDir: dataFullPath,
},
},
{
name: "no data dir",
Input: &Config{
input: &Config{
ConfigPaths: &ConfigurationPaths{
ConfigDir: "./tests",
ConfigDir: "./testdata",
HubDir: "./hub",
HubIndexFile: "./hub/.index.json",
},
},
expectedResult: nil,
expectedErr: "please provide a data directory with the 'data_dir' directive in the 'config_paths' section",
},
{
name: "no configuration path",
Input: &Config{},
expectedResult: nil,
name: "no configuration path",
input: &Config{},
expectedErr: "no configuration paths provided",
},
}
for idx, test := range tests {
err := test.Input.LoadHub()
if err == nil && test.err != "" {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
} else if test.err != "" {
if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
test.err,
fmt.Sprintf("%s", err))
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
err := tc.input.LoadHub()
cstest.RequireErrorContains(t, err, tc.expectedErr)
if tc.expectedErr != "" {
return
}
}
isOk := assert.Equal(t, test.expectedResult, test.Input.Hub)
if !isOk {
t.Fatalf("TEST '%s': NOK", test.name)
} else {
fmt.Printf("TEST '%s': OK\n", test.name)
}
assert.Equal(t, tc.expected, tc.input.Hub)
})
}
}

4
pkg/csconfig/profiles.go
View file

@ -6,10 +6,11 @@ import (
"fmt"
"io"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/yamlpatch"
"github.com/crowdsecurity/crowdsec/pkg/models"
"gopkg.in/yaml.v2"
)
// var OnErrorDefault = OnErrorIgnore
@ -43,7 +44,6 @@ func (c *LocalApiServerCfg) LoadProfiles() error {
}
reader := bytes.NewReader(fcontent)
//process the yaml
dec := yaml.NewDecoder(reader)
dec.SetStrict(true)
for {

12
pkg/csconfig/prometheus_test.go
View file

@ -3,21 +3,21 @@ package csconfig
import (
"testing"
"github.com/crowdsecurity/go-cs-lib/cstest"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/cstest"
)
func TestLoadPrometheus(t *testing.T) {
tests := []struct {
name string
Input *Config
input *Config
expectedURL string
expectedErr string
}{
{
name: "basic valid configuration",
Input: &Config{
input: &Config{
Prometheus: &PrometheusCfg{
Enabled: true,
Level: "full",
@ -33,10 +33,10 @@ func TestLoadPrometheus(t *testing.T) {
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
err := tc.Input.LoadPrometheus()
err := tc.input.LoadPrometheus()
cstest.RequireErrorContains(t, err, tc.expectedErr)
require.Equal(t, tc.expectedURL, tc.Input.Cscli.PrometheusUrl)
require.Equal(t, tc.expectedURL, tc.input.Cscli.PrometheusUrl)
})
}
}

Some files were not shown because too many files have changed in this diff Show more