diff --git a/.github/governance.yml b/.github/governance.yml index c57fd3626..0457a24c7 100644 --- a/.github/governance.yml +++ b/.github/governance.yml @@ -42,7 +42,7 @@ issue: 3. Check [Releases](https://github.com/crowdsecurity/crowdsec/releases/latest) to make sure your agent is on the latest version. - prefix: kind - list: ['feature', 'bug', 'packaging', 'enhancement'] + list: ['feature', 'bug', 'packaging', 'enhancement', 'refactoring'] multiple: false author_association: author: true @@ -54,6 +54,7 @@ issue: @$AUTHOR: There are no 'kind' label on this issue. You need a 'kind' label to start the triage process. * `/kind feature` * `/kind enhancement` + * `/kind refactoring` * `/kind bug` * `/kind packaging` @@ -65,12 +66,13 @@ pull_request: labels: - prefix: kind multiple: false - list: [ 'feature', 'enhancement', 'fix', 'chore', 'dependencies'] + list: [ 'feature', 'enhancement', 'fix', 'chore', 'dependencies', 'refactoring'] needs: comment: | @$AUTHOR: There are no 'kind' label on this PR. You need a 'kind' label to generate the release automatically. * `/kind feature` * `/kind enhancement` + * `/kind refactoring` * `/kind fix` * `/kind chore` * `/kind dependencies` diff --git a/.github/workflows/bats-sqlite-coverage.yml b/.github/workflows/bats-sqlite-coverage.yml index d56d69f28..742d1ee65 100644 --- a/.github/workflows/bats-sqlite-coverage.yml +++ b/.github/workflows/bats-sqlite-coverage.yml @@ -81,3 +81,4 @@ jobs: with: files: ./coverage-bats.out flags: bats + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml index d3ae4f90d..3e87d3ba4 100644 --- a/.github/workflows/docker-tests.yml +++ b/.github/workflows/docker-tests.yml @@ -59,15 +59,15 @@ jobs: cd docker/test python -m pip install --upgrade pipenv wheel - - name: "Cache virtualenvs" - id: cache-pipenv - uses: actions/cache@v4 - with: - path: ~/.local/share/virtualenvs - key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }} + #- name: "Cache virtualenvs" + # id: cache-pipenv + # uses: actions/cache@v4 + # with: + # path: ~/.local/share/virtualenvs + # key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }} - name: "Install dependencies" - if: steps.cache-pipenv.outputs.cache-hit != 'true' + #if: steps.cache-pipenv.outputs.cache-hit != 'true' run: | cd docker/test pipenv install --deploy diff --git a/.github/workflows/go-tests-windows.yml b/.github/workflows/go-tests-windows.yml index 781f2a4a9..9d5c17397 100644 --- a/.github/workflows/go-tests-windows.yml +++ b/.github/workflows/go-tests-windows.yml @@ -52,6 +52,7 @@ jobs: with: files: coverage.out flags: unit-windows + token: ${{ secrets.CODECOV_TOKEN }} - name: golangci-lint uses: golangci/golangci-lint-action@v4 diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 67f73d81a..4eac3777d 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -153,6 +153,7 @@ jobs: with: files: coverage.out flags: unit-linux + token: ${{ secrets.CODECOV_TOKEN }} - name: golangci-lint uses: golangci/golangci-lint-action@v4 diff --git a/.golangci.yml b/.golangci.yml index df0cb67d1..ff46ef1c0 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -37,17 +37,10 @@ linters-settings: statements: 122 govet: - enable: - - atomicalign - - deepequalerrors - # TODO: - fieldalignment - - findcall - - nilness - # TODO: - reflectvaluecompare - - shadow - - sortslice - - timeformat - - unusedwrite + enable-all: true + disable: + - reflectvaluecompare + - fieldalignment lll: # lower this after refactoring @@ -65,7 +58,7 @@ linters-settings: min-complexity: 28 nlreturn: - block-size: 4 + block-size: 5 nolintlint: allow-unused: false # report any unused nolint directives @@ -147,37 +140,40 @@ linters: # # DEPRECATED by golangi-lint # - - deadcode # The owner seems to have abandoned the linter. Replaced by unused. - - exhaustivestruct # The owner seems to have abandoned the linter. Replaced by exhaustruct. - - golint # Golint differs from gofmt. Gofmt reformats Go source code, whereas golint prints out style mistakes - - ifshort # Checks that your code uses short syntax for if-statements whenever possible - - interfacer # Linter that suggests narrower interface types - - maligned # Tool to detect Go structs that would take less memory if their fields were sorted - - nosnakecase # nosnakecase is a linter that detects snake case of variable naming and function name. - - scopelint # Scopelint checks for unpinned variables in go programs - - structcheck # The owner seems to have abandoned the linter. Replaced by unused. - - varcheck # The owner seems to have abandoned the linter. Replaced by unused. + - deadcode + - exhaustivestruct + - golint + - ifshort + - interfacer + - maligned + - nosnakecase + - scopelint + - structcheck + - varcheck # # Enabled # # - asasalint # check for pass []any as any in variadic func(...any) - # - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers + # - asciicheck # checks that all code identifiers does not have non-ASCII symbols in the name # - bidichk # Checks for dangerous unicode character sequences # - bodyclose # checks whether HTTP response body is closed successfully + # - copyloopvar # copyloopvar is a linter detects places where loop variables are copied # - cyclop # checks function and package cyclomatic complexity # - decorder # check declaration order and count of types, constants, variables and functions # - depguard # Go linter that checks if package imports are in a list of acceptable packages # - dupword # checks for duplicate words in the source code # - durationcheck # check for two durations multiplied together - # - errcheck # Errcheck is a program for checking for unchecked errors in go programs. These unchecked errors can be critical bugs in some cases + # - errcheck # errcheck is a program for checking for unchecked errors in Go code. These unchecked errors can be critical bugs in some cases # - errorlint # errorlint is a linter for that can be used to find code that will cause problems with the error wrapping scheme introduced in Go 1.13. # - execinquery # execinquery is a linter about query string checker in Query function which reads your Go src files and warning it finds # - exportloopref # checks for pointers to enclosing loop variables # - funlen # Tool for detection of long functions # - ginkgolinter # enforces standards of using ginkgo and gomega + # - gocheckcompilerdirectives # Checks that go compiler directive comments (//go:) are valid. # - gochecknoinits # Checks that no init functions are present in Go code + # - gochecksumtype # Run exhaustiveness checks on Go "sum types" # - gocognit # Computes and checks the cognitive complexity of functions # - gocritic # Provides diagnostics that check for bugs, performance and style issues. # - gocyclo # Computes and checks the cyclomatic complexity of functions @@ -185,48 +181,56 @@ linters: # - gomoddirectives # Manage the use of 'replace', 'retract', and 'excludes' directives in go.mod. # - gomodguard # Allow and block list linter for direct Go module dependencies. This is different from depguard where there are different block types for example version constraints and module recommendations. # - goprintffuncname # Checks that printf-like functions are named with `f` at the end - # - gosimple # (megacheck): Linter for Go source code that specializes in simplifying a code - # - govet # (vet, vetshadow): Vet examines Go source code and reports suspicious constructs, such as Printf calls whose arguments do not align with the format string - # - grouper # An analyzer to analyze expression groups. + # - gosimple # (megacheck): Linter for Go source code that specializes in simplifying code + # - gosmopolitan # Report certain i18n/l10n anti-patterns in your Go codebase + # - govet # (vet, vetshadow): Vet examines Go source code and reports suspicious constructs. It is roughly the same as 'go vet' and uses its passes. + # - grouper # Analyze expression groups. # - importas # Enforces consistent import aliases # - ineffassign # Detects when assignments to existing variables are not used # - interfacebloat # A linter that checks the number of methods inside an interface. + # - intrange # intrange is a linter to find places where for loops could make use of an integer range. # - lll # Reports long lines + # - loggercheck # (logrlint): Checks key value pairs for common logger libraries (kitlog,klog,logr,zap). # - logrlint # Check logr arguments. # - maintidx # maintidx measures the maintainability index of each function. # - makezero # Finds slice declarations with non-zero initial length - # - misspell # Finds commonly misspelled English words in comments - # - nakedret # Finds naked returns in functions greater than a specified function length + # - mirror # reports wrong mirror patterns of bytes/strings usage + # - misspell # Finds commonly misspelled English words + # - nakedret # Checks that functions with naked returns are not longer than a maximum size (can be zero). # - nestif # Reports deeply nested if statements # - nilerr # Finds the code that returns nil even if it checks that the error is not nil. # - nolintlint # Reports ill-formed or insufficient nolint directives # - nonamedreturns # Reports all named returns # - nosprintfhostport # Checks for misuse of Sprintf to construct a host with port in a URL. + # - perfsprint # Checks that fmt.Sprintf can be replaced with a faster alternative. # - predeclared # find code that shadows one of Go's predeclared identifiers # - reassign # Checks that package variables are not reassigned - # - rowserrcheck # checks whether Err of rows is checked successfully - # - sqlclosecheck # Checks that sql.Rows and sql.Stmt are closed. - # - staticcheck # (megacheck): Staticcheck is a go vet on steroids, applying a ton of static analysis checks - # - testableexamples # linter checks if examples are testable (have an expected output) + # - rowserrcheck # checks whether Rows.Err of rows is checked successfully + # - sloglint # ensure consistent code style when using log/slog + # - spancheck # Checks for mistakes with OpenTelemetry/Census spans. + # - sqlclosecheck # Checks that sql.Rows, sql.Stmt, sqlx.NamedStmt, pgx.Query are closed. + # - staticcheck # (megacheck): It's a set of rules from staticcheck. It's not the same thing as the staticcheck binary. The author of staticcheck doesn't support or approve the use of staticcheck as a library inside golangci-lint. # - tenv # tenv is analyzer that detects using os.Setenv instead of t.Setenv since Go1.17 + # - testableexamples # linter checks if examples are testable (have an expected output) + # - testifylint # Checks usage of github.com/stretchr/testify. # - tparallel # tparallel detects inappropriate usage of t.Parallel() method in your Go test codes - # - typecheck # Like the front-end of a Go compiler, parses and type-checks Go code # - unconvert # Remove unnecessary type conversions # - unused # (megacheck): Checks Go code for unused constants, variables, functions and types # - usestdlibvars # A linter that detect the possibility to use variables/constants from the Go standard library. - # - wastedassign # wastedassign finds wasted assignment statements. + # - wastedassign # Finds wasted assignment statements + # - zerologlint # Detects the wrong usage of `zerolog` that a user forgets to dispatch with `Send` or `Msg` # # Recommended? (easy) # - dogsled # Checks assignments with too many blank identifiers (e.g. x, _, _, _, := f()) - - errchkjson # Checks types passed to the json encoding functions. Reports unsupported types and optionally reports occations, where the check for the returned error can be omitted. + - errchkjson # Checks types passed to the json encoding functions. Reports unsupported types and reports occations, where the check for the returned error can be omitted. - exhaustive # check exhaustiveness of enum switch statements - gci # Gci control golang package import order and make it always deterministic. - godot # Check if comments end in a period - gofmt # Gofmt checks whether code was gofmt-ed. By default this tool runs with -s option to check for code simplification - - goimports # In addition to fixing imports, goimports also formats your code in the same style as gofmt. + - goimports # Check import statements are formatted according to the 'goimport' command. Reformat imports in autofix mode. - gosec # (gas): Inspects source code for security problems - inamedparam # reports interfaces with unnamed method parameters - musttag # enforce field tags in (un)marshaled structs @@ -234,7 +238,7 @@ linters: - protogetter # Reports direct reads from proto message fields when getters should be used - revive # Fast, configurable, extensible, flexible, and beautiful linter for Go. Drop-in replacement of golint. - tagalign # check that struct tags are well aligned - - thelper # thelper detects golang test helpers without t.Helper() call and checks the consistency of test helpers + - thelper # thelper detects tests helpers which is not start with t.Helper() method. - wrapcheck # Checks that errors returned from external packages are wrapped # @@ -242,12 +246,12 @@ linters: # - containedctx # containedctx is a linter that detects struct contained context.Context field - - contextcheck # check the function whether use a non-inherited context + - contextcheck # check whether the function uses a non-inherited context - errname # Checks that sentinel errors are prefixed with the `Err` and error types are suffixed with the `Error`. - gomnd # An analyzer to detect magic numbers. - ireturn # Accept Interfaces, Return Concrete Types - nilnil # Checks that there is no simultaneous return of `nil` error and an invalid value. - - noctx # noctx finds sending http request without context.Context + - noctx # Finds sending http request without context.Context - unparam # Reports unused function parameters # @@ -256,8 +260,8 @@ linters: - gofumpt # Gofumpt checks whether code was gofumpt-ed. - nlreturn # nlreturn checks for a new line before return and branch statements to increase code clarity - - whitespace # Tool for detection of leading and trailing whitespace - - wsl # Whitespace Linter - Forces you to use empty lines! + - whitespace # Whitespace is a linter that checks for unnecessary newlines at the start and end of functions, if, for, etc. + - wsl # add or remove empty lines # # Well intended, but not ready for this @@ -265,8 +269,8 @@ linters: - dupl # Tool for code clone detection - forcetypeassert # finds forced type assertions - godox # Tool for detection of FIXME, TODO and other comment keywords - - goerr113 # Golang linter to check the errors handling expressions - - paralleltest # paralleltest detects missing usage of t.Parallel() method in your Go test + - goerr113 # Go linter to check the errors handling expressions + - paralleltest # Detects missing usage of t.Parallel() method in your Go test - testpackage # linter that makes you use a separate _test package # @@ -274,7 +278,7 @@ linters: # - exhaustruct # Checks if all structure fields are initialized - forbidigo # Forbids identifiers - - gochecknoglobals # check that no global variables exist + - gochecknoglobals # Check that no global variables exist. - goconst # Finds repeated strings that could be replaced by a constant - stylecheck # Stylecheck is a replacement for golint - tagliatelle # Checks the struct tags. diff --git a/Dockerfile b/Dockerfile index 1e311bfa8..69de0f9df 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold core cd re2-${RE2_VERSION} && \ make install && \ echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \ - go install github.com/mikefarah/yq/v4@v4.40.4 + go install github.com/mikefarah/yq/v4@v4.43.1 COPY . . diff --git a/Dockerfile.debian b/Dockerfile.debian index ee0fa32bb..9bcb517bb 100644 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -21,7 +21,7 @@ RUN apt-get update && \ make && \ make install && \ echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \ - go install github.com/mikefarah/yq/v4@v4.40.4 + go install github.com/mikefarah/yq/v4@v4.43.1 COPY . . diff --git a/cmd/crowdsec-cli/alerts.go b/cmd/crowdsec-cli/alerts.go index 908466f9e..59dff8094 100644 --- a/cmd/crowdsec-cli/alerts.go +++ b/cmd/crowdsec-cli/alerts.go @@ -4,6 +4,7 @@ import ( "context" "encoding/csv" "encoding/json" + "errors" "fmt" "net/url" "os" @@ -204,6 +205,7 @@ func (cli *cliAlerts) NewCommand() *cobra.Command { if err != nil { return fmt.Errorf("parsing api url %s: %w", apiURL, err) } + cli.client, err = apiclient.NewClient(&apiclient.Config{ MachineID: cfg.API.Client.Credentials.Login, Password: strfmt.Password(cfg.API.Client.Credentials.Password), @@ -211,7 +213,6 @@ func (cli *cliAlerts) NewCommand() *cobra.Command { URL: apiURL, VersionPrefix: "v1", }) - if err != nil { return fmt.Errorf("new api client: %w", err) } @@ -229,7 +230,7 @@ func (cli *cliAlerts) NewCommand() *cobra.Command { } func (cli *cliAlerts) NewListCmd() *cobra.Command { - var alertListFilter = apiclient.AlertsListOpts{ + alertListFilter := apiclient.AlertsListOpts{ ScopeEquals: new(string), ValueEquals: new(string), ScenarioEquals: new(string), @@ -363,7 +364,7 @@ func (cli *cliAlerts) NewDeleteCmd() *cobra.Command { delAlertByID string ) - var alertDeleteFilter = apiclient.AlertsDeleteOpts{ + alertDeleteFilter := apiclient.AlertsDeleteOpts{ ScopeEquals: new(string), ValueEquals: new(string), ScenarioEquals: new(string), @@ -391,7 +392,7 @@ cscli alerts delete -s crowdsecurity/ssh-bf"`, *alertDeleteFilter.ScenarioEquals == "" && *alertDeleteFilter.IPEquals == "" && *alertDeleteFilter.RangeEquals == "" && delAlertByID == "" { _ = cmd.Usage() - return fmt.Errorf("at least one filter or --all must be specified") + return errors.New("at least one filter or --all must be specified") } return nil @@ -478,7 +479,7 @@ func (cli *cliAlerts) NewInspectCmd() *cobra.Command { cfg := cli.cfg() if len(args) == 0 { printHelp(cmd) - return fmt.Errorf("missing alert_id") + return errors.New("missing alert_id") } for _, alertID := range args { id, err := strconv.Atoi(alertID) diff --git a/cmd/crowdsec-cli/capi.go b/cmd/crowdsec-cli/capi.go index e56a8a747..ec11acbbe 100644 --- a/cmd/crowdsec-cli/capi.go +++ b/cmd/crowdsec-cli/capi.go @@ -175,7 +175,7 @@ func (cli *cliCapi) status() error { return err } - scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledNamesByType(cwhub.SCENARIOS) if err != nil { return fmt.Errorf("failed to get scenarios: %w", err) } diff --git a/cmd/crowdsec-cli/config_show.go b/cmd/crowdsec-cli/config_show.go index c277173c3..c7138c98e 100644 --- a/cmd/crowdsec-cli/config_show.go +++ b/cmd/crowdsec-cli/config_show.go @@ -10,13 +10,15 @@ import ( "github.com/sanity-io/litter" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" ) -func showConfigKey(key string) error { +func (cli *cliConfig) showKey(key string) error { + cfg := cli.cfg() + type Env struct { Config *csconfig.Config } @@ -30,15 +32,15 @@ func showConfigKey(key string) error { return err } - output, err := expr.Run(program, Env{Config: csConfig}) + output, err := expr.Run(program, Env{Config: cfg}) if err != nil { return err } - switch csConfig.Cscli.Output { + switch cfg.Cscli.Output { case "human", "raw": // Don't use litter for strings, it adds quotes - // that we didn't have before + // that would break compatibility with previous versions switch output.(type) { case string: fmt.Println(output) @@ -51,13 +53,14 @@ func showConfigKey(key string) error { return fmt.Errorf("failed to marshal configuration: %w", err) } - fmt.Printf("%s\n", string(data)) + fmt.Println(string(data)) } return nil } -var configShowTemplate = `Global: +func (cli *cliConfig) template() string { + return `Global: {{- if .ConfigPaths }} - Configuration Folder : {{.ConfigPaths.ConfigDir}} @@ -182,19 +185,11 @@ Central API: {{- end }} {{- end }} ` +} -func (cli *cliConfig) show(key string) error { +func (cli *cliConfig) show() error { cfg := cli.cfg() - if err := cfg.LoadAPIClient(); err != nil { - log.Errorf("failed to load API client configuration: %s", err) - // don't return, we can still show the configuration - } - - if key != "" { - return showConfigKey(key) - } - switch cfg.Cscli.Output { case "human": // The tests on .Enable look funny because the option has a true default which has @@ -205,7 +200,7 @@ func (cli *cliConfig) show(key string) error { "ValueBool": func(b *bool) bool { return b != nil && *b }, } - tmp, err := template.New("config").Funcs(funcs).Parse(configShowTemplate) + tmp, err := template.New("config").Funcs(funcs).Parse(cli.template()) if err != nil { return err } @@ -220,14 +215,14 @@ func (cli *cliConfig) show(key string) error { return fmt.Errorf("failed to marshal configuration: %w", err) } - fmt.Printf("%s\n", string(data)) + fmt.Println(string(data)) case "raw": data, err := yaml.Marshal(cfg) if err != nil { return fmt.Errorf("failed to marshal configuration: %w", err) } - fmt.Printf("%s\n", string(data)) + fmt.Println(string(data)) } return nil @@ -243,7 +238,16 @@ func (cli *cliConfig) newShowCmd() *cobra.Command { Args: cobra.ExactArgs(0), DisableAutoGenTag: true, RunE: func(_ *cobra.Command, _ []string) error { - return cli.show(key) + if err := cli.cfg().LoadAPIClient(); err != nil { + log.Errorf("failed to load API client configuration: %s", err) + // don't return, we can still show the configuration + } + + if key != "" { + return cli.showKey(key) + } + + return cli.show() }, } diff --git a/cmd/crowdsec-cli/console.go b/cmd/crowdsec-cli/console.go index b1912825c..4aba689aa 100644 --- a/cmd/crowdsec-cli/console.go +++ b/cmd/crowdsec-cli/console.go @@ -4,9 +4,11 @@ import ( "context" "encoding/csv" "encoding/json" + "errors" "fmt" "net/url" "os" + "strconv" "strings" "github.com/fatih/color" @@ -36,7 +38,7 @@ func NewCLIConsole(cfg configGetter) *cliConsole { } func (cli *cliConsole) NewCommand() *cobra.Command { - var cmd = &cobra.Command{ + cmd := &cobra.Command{ Use: "console [action]", Short: "Manage interaction with Crowdsec console (https://app.crowdsec.net)", Args: cobra.MinimumNArgs(1), @@ -101,7 +103,7 @@ After running this command your will need to validate the enrollment in the weba return err } - scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledNamesByType(cwhub.SCENARIOS) if err != nil { return fmt.Errorf("failed to get installed scenarios: %w", err) } @@ -203,7 +205,7 @@ Enable given information push to the central API. Allows to empower the console` log.Infof("All features have been enabled successfully") } else { if len(args) == 0 { - return fmt.Errorf("you must specify at least one feature to enable") + return errors.New("you must specify at least one feature to enable") } if err := cli.setConsoleOpts(args, true); err != nil { return err @@ -288,11 +290,11 @@ func (cli *cliConsole) newStatusCmd() *cobra.Command { } rows := [][]string{ - {csconfig.SEND_MANUAL_SCENARIOS, fmt.Sprintf("%t", *consoleCfg.ShareManualDecisions)}, - {csconfig.SEND_CUSTOM_SCENARIOS, fmt.Sprintf("%t", *consoleCfg.ShareCustomScenarios)}, - {csconfig.SEND_TAINTED_SCENARIOS, fmt.Sprintf("%t", *consoleCfg.ShareTaintedScenarios)}, - {csconfig.SEND_CONTEXT, fmt.Sprintf("%t", *consoleCfg.ShareContext)}, - {csconfig.CONSOLE_MANAGEMENT, fmt.Sprintf("%t", *consoleCfg.ConsoleManagement)}, + {csconfig.SEND_MANUAL_SCENARIOS, strconv.FormatBool(*consoleCfg.ShareManualDecisions)}, + {csconfig.SEND_CUSTOM_SCENARIOS, strconv.FormatBool(*consoleCfg.ShareCustomScenarios)}, + {csconfig.SEND_TAINTED_SCENARIOS, strconv.FormatBool(*consoleCfg.ShareTaintedScenarios)}, + {csconfig.SEND_CONTEXT, strconv.FormatBool(*consoleCfg.ShareContext)}, + {csconfig.CONSOLE_MANAGEMENT, strconv.FormatBool(*consoleCfg.ConsoleManagement)}, } for _, row := range rows { err = csvwriter.Write(row) diff --git a/cmd/crowdsec-cli/copyfile.go b/cmd/crowdsec-cli/copyfile.go index 332f744be..272fb3f78 100644 --- a/cmd/crowdsec-cli/copyfile.go +++ b/cmd/crowdsec-cli/copyfile.go @@ -9,7 +9,6 @@ import ( log "github.com/sirupsen/logrus" ) - /*help to copy the file, ioutil doesn't offer the feature*/ func copyFileContents(src, dst string) (err error) { @@ -69,6 +68,7 @@ func CopyFile(sourceSymLink, destinationFile string) error { if !(destinationFileStat.Mode().IsRegular()) { return fmt.Errorf("copyFile: non-regular destination file %s (%q)", destinationFileStat.Name(), destinationFileStat.Mode().String()) } + if os.SameFile(sourceFileStat, destinationFileStat) { return err } @@ -80,4 +80,3 @@ func CopyFile(sourceSymLink, destinationFile string) error { return err } - diff --git a/cmd/crowdsec-cli/decisions.go b/cmd/crowdsec-cli/decisions.go index a97536ddc..3fb790633 100644 --- a/cmd/crowdsec-cli/decisions.go +++ b/cmd/crowdsec-cli/decisions.go @@ -4,6 +4,7 @@ import ( "context" "encoding/csv" "encoding/json" + "errors" "fmt" "net/url" "os" @@ -346,7 +347,7 @@ cscli decisions add --scope username --value foobar addScope = types.Range } else if addValue == "" { printHelp(cmd) - return fmt.Errorf("missing arguments, a value is required (--ip, --range or --scope and --value)") + return errors.New("missing arguments, a value is required (--ip, --range or --scope and --value)") } if addReason == "" { @@ -371,7 +372,7 @@ cscli decisions add --scope username --value foobar Scenario: &addReason, ScenarioVersion: &empty, Simulated: &simulated, - //setting empty scope/value broke plugins, and it didn't seem to be needed anymore w/ latest papi changes + // setting empty scope/value broke plugins, and it didn't seem to be needed anymore w/ latest papi changes Source: &models.Source{ AsName: empty, AsNumber: empty, @@ -411,7 +412,7 @@ cscli decisions add --scope username --value foobar } func (cli *cliDecisions) newDeleteCmd() *cobra.Command { - var delFilter = apiclient.DecisionsDeleteOpts{ + delFilter := apiclient.DecisionsDeleteOpts{ ScopeEquals: new(string), ValueEquals: new(string), TypeEquals: new(string), @@ -448,7 +449,7 @@ cscli decisions delete --origin lists --scenario list_name *delFilter.RangeEquals == "" && *delFilter.ScenarioEquals == "" && *delFilter.OriginEquals == "" && delDecisionID == "" { cmd.Usage() - return fmt.Errorf("at least one filter or --all must be specified") + return errors.New("at least one filter or --all must be specified") } return nil diff --git a/cmd/crowdsec-cli/decisions_import.go b/cmd/crowdsec-cli/decisions_import.go index 45d1841a6..8c36bd5dc 100644 --- a/cmd/crowdsec-cli/decisions_import.go +++ b/cmd/crowdsec-cli/decisions_import.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "os" @@ -81,7 +82,7 @@ func (cli *cliDecisions) runImport(cmd *cobra.Command, args []string) error { } if defaultDuration == "" { - return fmt.Errorf("--duration cannot be empty") + return errors.New("--duration cannot be empty") } defaultScope, err := flags.GetString("scope") @@ -90,7 +91,7 @@ func (cli *cliDecisions) runImport(cmd *cobra.Command, args []string) error { } if defaultScope == "" { - return fmt.Errorf("--scope cannot be empty") + return errors.New("--scope cannot be empty") } defaultReason, err := flags.GetString("reason") @@ -99,7 +100,7 @@ func (cli *cliDecisions) runImport(cmd *cobra.Command, args []string) error { } if defaultReason == "" { - return fmt.Errorf("--reason cannot be empty") + return errors.New("--reason cannot be empty") } defaultType, err := flags.GetString("type") @@ -108,7 +109,7 @@ func (cli *cliDecisions) runImport(cmd *cobra.Command, args []string) error { } if defaultType == "" { - return fmt.Errorf("--type cannot be empty") + return errors.New("--type cannot be empty") } batchSize, err := flags.GetInt("batch") @@ -136,7 +137,7 @@ func (cli *cliDecisions) runImport(cmd *cobra.Command, args []string) error { } if format == "" { - return fmt.Errorf("unable to guess format from file extension, please provide a format with --format flag") + return errors.New("unable to guess format from file extension, please provide a format with --format flag") } if input == "-" { @@ -235,7 +236,6 @@ func (cli *cliDecisions) runImport(cmd *cobra.Command, args []string) error { return nil } - func (cli *cliDecisions) newImportCmd() *cobra.Command { cmd := &cobra.Command{ Use: "import [options]", diff --git a/cmd/crowdsec-cli/doc.go b/cmd/crowdsec-cli/doc.go index a4896f3da..4b1d50d15 100644 --- a/cmd/crowdsec-cli/doc.go +++ b/cmd/crowdsec-cli/doc.go @@ -39,8 +39,10 @@ id: %s title: %s --- ` + name := filepath.Base(filename) base := strings.TrimSuffix(name, filepath.Ext(name)) + return fmt.Sprintf(header, base, strings.ReplaceAll(base, "_", " ")) } diff --git a/cmd/crowdsec-cli/explain.go b/cmd/crowdsec-cli/explain.go index ce323fd0c..c322cce47 100644 --- a/cmd/crowdsec-cli/explain.go +++ b/cmd/crowdsec-cli/explain.go @@ -83,7 +83,7 @@ tail -n 5 myfile.log | cscli explain --type nginx -f - PersistentPreRunE: func(_ *cobra.Command, _ []string) error { fileInfo, _ := os.Stdin.Stat() if cli.flags.logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) { - return fmt.Errorf("the option -f - is intended to work with pipes") + return errors.New("the option -f - is intended to work with pipes") } return nil @@ -160,18 +160,22 @@ func (cli *cliExplain) run() error { } else if logFile == "-" { reader := bufio.NewReader(os.Stdin) errCount := 0 + for { input, err := reader.ReadBytes('\n') if err != nil && errors.Is(err, io.EOF) { break } + if len(input) > 1 { _, err = f.Write(input) } + if err != nil || len(input) <= 1 { errCount++ } } + if errCount > 0 { log.Warnf("Failed to write %d lines to %s", errCount, tmpFile) } @@ -207,7 +211,7 @@ func (cli *cliExplain) run() error { } if dsn == "" { - return fmt.Errorf("no acquisition (--file or --dsn) provided, can't run cscli test") + return errors.New("no acquisition (--file or --dsn) provided, can't run cscli test") } cmdArgs := []string{"-c", ConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", dir, "-no-api"} diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index 600e56889..71347a5a5 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -13,7 +13,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -type cliHub struct { +type cliHub struct{ cfg configGetter } @@ -137,7 +137,7 @@ func (cli *cliHub) upgrade(force bool) error { } for _, itemType := range cwhub.ItemTypes { - items, err := hub.GetInstalledItems(itemType) + items, err := hub.GetInstalledItemsByType(itemType) if err != nil { return err } diff --git a/cmd/crowdsec-cli/hubappsec.go b/cmd/crowdsec-cli/hubappsec.go index ff41ad5f9..7ee578edc 100644 --- a/cmd/crowdsec-cli/hubappsec.go +++ b/cmd/crowdsec-cli/hubappsec.go @@ -13,8 +13,9 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -func NewCLIAppsecConfig() *cliItem { +func NewCLIAppsecConfig(cfg configGetter) *cliItem { return &cliItem{ + cfg: cfg, name: cwhub.APPSEC_CONFIGS, singular: "appsec-config", oneOrMore: "appsec-config(s)", @@ -46,7 +47,7 @@ cscli appsec-configs list crowdsecurity/vpatch`, } } -func NewCLIAppsecRule() *cliItem { +func NewCLIAppsecRule(cfg configGetter) *cliItem { inspectDetail := func(item *cwhub.Item) error { // Only show the converted rules in human mode if csConfig.Cscli.Output != "human" { @@ -57,11 +58,11 @@ func NewCLIAppsecRule() *cliItem { yamlContent, err := os.ReadFile(item.State.LocalPath) if err != nil { - return fmt.Errorf("unable to read file %s : %s", item.State.LocalPath, err) + return fmt.Errorf("unable to read file %s: %w", item.State.LocalPath, err) } if err := yaml.Unmarshal(yamlContent, &appsecRule); err != nil { - return fmt.Errorf("unable to unmarshal yaml file %s : %s", item.State.LocalPath, err) + return fmt.Errorf("unable to unmarshal yaml file %s: %w", item.State.LocalPath, err) } for _, ruleType := range appsec_rule.SupportedTypes() { @@ -70,7 +71,7 @@ func NewCLIAppsecRule() *cliItem { for _, rule := range appsecRule.Rules { convertedRule, _, err := rule.Convert(ruleType, appsecRule.Name) if err != nil { - return fmt.Errorf("unable to convert rule %s : %s", rule.Name, err) + return fmt.Errorf("unable to convert rule %s: %w", rule.Name, err) } fmt.Println(convertedRule) @@ -88,6 +89,7 @@ func NewCLIAppsecRule() *cliItem { } return &cliItem{ + cfg: cfg, name: "appsec-rules", singular: "appsec-rule", oneOrMore: "appsec-rule(s)", diff --git a/cmd/crowdsec-cli/hubcollection.go b/cmd/crowdsec-cli/hubcollection.go index dee9a0b9e..655b36eb1 100644 --- a/cmd/crowdsec-cli/hubcollection.go +++ b/cmd/crowdsec-cli/hubcollection.go @@ -4,8 +4,9 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -func NewCLICollection() *cliItem { +func NewCLICollection(cfg configGetter) *cliItem { return &cliItem{ + cfg: cfg, name: cwhub.COLLECTIONS, singular: "collection", oneOrMore: "collection(s)", diff --git a/cmd/crowdsec-cli/hubcontext.go b/cmd/crowdsec-cli/hubcontext.go index 630dbb2f7..2a7773273 100644 --- a/cmd/crowdsec-cli/hubcontext.go +++ b/cmd/crowdsec-cli/hubcontext.go @@ -4,8 +4,9 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -func NewCLIContext() *cliItem { +func NewCLIContext(cfg configGetter) *cliItem { return &cliItem{ + cfg: cfg, name: cwhub.CONTEXTS, singular: "context", oneOrMore: "context(s)", diff --git a/cmd/crowdsec-cli/hubparser.go b/cmd/crowdsec-cli/hubparser.go index 0b224c8a7..cc856cbed 100644 --- a/cmd/crowdsec-cli/hubparser.go +++ b/cmd/crowdsec-cli/hubparser.go @@ -4,8 +4,9 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -func NewCLIParser() *cliItem { +func NewCLIParser(cfg configGetter) *cliItem { return &cliItem{ + cfg: cfg, name: cwhub.PARSERS, singular: "parser", oneOrMore: "parser(s)", diff --git a/cmd/crowdsec-cli/hubpostoverflow.go b/cmd/crowdsec-cli/hubpostoverflow.go index 908ccbea0..3fd45fd11 100644 --- a/cmd/crowdsec-cli/hubpostoverflow.go +++ b/cmd/crowdsec-cli/hubpostoverflow.go @@ -4,8 +4,9 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -func NewCLIPostOverflow() *cliItem { +func NewCLIPostOverflow(cfg configGetter) *cliItem { return &cliItem{ + cfg: cfg, name: cwhub.POSTOVERFLOWS, singular: "postoverflow", oneOrMore: "postoverflow(s)", diff --git a/cmd/crowdsec-cli/hubscenario.go b/cmd/crowdsec-cli/hubscenario.go index 1de2182bf..4434b9a2c 100644 --- a/cmd/crowdsec-cli/hubscenario.go +++ b/cmd/crowdsec-cli/hubscenario.go @@ -4,8 +4,9 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -func NewCLIScenario() *cliItem { +func NewCLIScenario(cfg configGetter) *cliItem { return &cliItem{ + cfg: cfg, name: cwhub.SCENARIOS, singular: "scenario", oneOrMore: "scenario(s)", diff --git a/cmd/crowdsec-cli/hubtest.go b/cmd/crowdsec-cli/hubtest.go index d6ed45600..51735ce19 100644 --- a/cmd/crowdsec-cli/hubtest.go +++ b/cmd/crowdsec-cli/hubtest.go @@ -135,6 +135,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios // create empty nuclei template file nucleiFileName := fmt.Sprintf("%s.yaml", testName) nucleiFilePath := filepath.Join(testPath, nucleiFileName) + nucleiFile, err := os.OpenFile(nucleiFilePath, os.O_RDWR|os.O_CREATE, 0755) if err != nil { return err @@ -405,7 +406,7 @@ func (cli *cliHubTest) NewRunCmd() *cobra.Command { } func (cli *cliHubTest) NewCleanCmd() *cobra.Command { - var cmd = &cobra.Command{ + cmd := &cobra.Command{ Use: "clean", Short: "clean [test_name]", Args: cobra.MinimumNArgs(1), diff --git a/cmd/crowdsec-cli/item_metrics.go b/cmd/crowdsec-cli/item_metrics.go index e6f27ae5d..b571fb1c5 100644 --- a/cmd/crowdsec-cli/item_metrics.go +++ b/cmd/crowdsec-cli/item_metrics.go @@ -37,6 +37,7 @@ func ShowMetrics(hubItem *cwhub.Item) error { appsecMetricsTable(color.Output, hubItem.Name, metrics) default: // no metrics for this item type } + return nil } @@ -49,21 +50,27 @@ func GetParserMetric(url string, itemName string) map[string]map[string]int { if !strings.HasPrefix(fam.Name, "cs_") { continue } + log.Tracef("round %d", idx) + for _, m := range fam.Metrics { metric, ok := m.(prom2json.Metric) if !ok { log.Debugf("failed to convert metric to prom2json.Metric") continue } + name, ok := metric.Labels["name"] if !ok { log.Debugf("no name in Metric %v", metric.Labels) } + if name != itemName { continue } + source, ok := metric.Labels["source"] + if !ok { log.Debugf("no source in Metric %v", metric.Labels) } else { @@ -71,12 +78,15 @@ func GetParserMetric(url string, itemName string) map[string]map[string]int { source = srctype + ":" + source } } + value := m.(prom2json.Metric).Value + fval, err := strconv.ParseFloat(value, 32) if err != nil { log.Errorf("Unexpected int value %s : %s", value, err) continue } + ival := int(fval) switch fam.Name { @@ -119,6 +129,7 @@ func GetParserMetric(url string, itemName string) map[string]map[string]int { } } } + return stats } @@ -136,26 +147,34 @@ func GetScenarioMetric(url string, itemName string) map[string]int { if !strings.HasPrefix(fam.Name, "cs_") { continue } + log.Tracef("round %d", idx) + for _, m := range fam.Metrics { metric, ok := m.(prom2json.Metric) if !ok { log.Debugf("failed to convert metric to prom2json.Metric") continue } + name, ok := metric.Labels["name"] + if !ok { log.Debugf("no name in Metric %v", metric.Labels) } + if name != itemName { continue } + value := m.(prom2json.Metric).Value + fval, err := strconv.ParseFloat(value, 32) if err != nil { log.Errorf("Unexpected int value %s : %s", value, err) continue } + ival := int(fval) switch fam.Name { @@ -174,6 +193,7 @@ func GetScenarioMetric(url string, itemName string) map[string]int { } } } + return stats } @@ -188,17 +208,22 @@ func GetAppsecRuleMetric(url string, itemName string) map[string]int { if !strings.HasPrefix(fam.Name, "cs_") { continue } + log.Tracef("round %d", idx) + for _, m := range fam.Metrics { metric, ok := m.(prom2json.Metric) if !ok { log.Debugf("failed to convert metric to prom2json.Metric") continue } + name, ok := metric.Labels["rule_name"] + if !ok { log.Debugf("no rule_name in Metric %v", metric.Labels) } + if name != itemName { continue } @@ -209,11 +234,13 @@ func GetAppsecRuleMetric(url string, itemName string) map[string]int { } value := m.(prom2json.Metric).Value + fval, err := strconv.ParseFloat(value, 32) if err != nil { log.Errorf("Unexpected int value %s : %s", value, err) continue } + ival := int(fval) switch fam.Name { @@ -231,6 +258,7 @@ func GetAppsecRuleMetric(url string, itemName string) map[string]int { } } } + return stats } @@ -247,6 +275,7 @@ func GetPrometheusMetric(url string) []*prom2json.Family { go func() { defer trace.CatchPanic("crowdsec/GetPrometheusMetric") + err := prom2json.FetchMetricFamilies(url, mfChan, transport) if err != nil { log.Fatalf("failed to fetch prometheus metrics : %v", err) @@ -257,6 +286,7 @@ func GetPrometheusMetric(url string) []*prom2json.Family { for mf := range mfChan { result = append(result, prom2json.NewFamily(mf)) } + log.Debugf("Finished reading prometheus output, %d entries", len(result)) return result diff --git a/cmd/crowdsec-cli/item_suggest.go b/cmd/crowdsec-cli/item_suggest.go index d3beee721..9272abedd 100644 --- a/cmd/crowdsec-cli/item_suggest.go +++ b/cmd/crowdsec-cli/item_suggest.go @@ -61,7 +61,7 @@ func compInstalledItems(itemType string, args []string, toComplete string) ([]st return nil, cobra.ShellCompDirectiveDefault } - items, err := hub.GetInstalledItemNames(itemType) + items, err := hub.GetInstalledNamesByType(itemType) if err != nil { cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true) return nil, cobra.ShellCompDirectiveDefault diff --git a/cmd/crowdsec-cli/itemcli.go b/cmd/crowdsec-cli/itemcli.go index 4f3dc40ae..cdaf78770 100644 --- a/cmd/crowdsec-cli/itemcli.go +++ b/cmd/crowdsec-cli/itemcli.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "os" "strings" @@ -28,6 +29,7 @@ type cliHelp struct { } type cliItem struct { + cfg configGetter name string // plural, as used in the hub index singular string oneOrMore string // parenthetical pluralizaion: "parser(s)" @@ -61,7 +63,9 @@ func (cli cliItem) NewCommand() *cobra.Command { } func (cli cliItem) install(args []string, downloadOnly bool, force bool, ignoreError bool) error { - hub, err := require.Hub(csConfig, require.RemoteHub(csConfig), log.StandardLogger()) + cfg := cli.cfg() + + hub, err := require.Hub(cfg, require.RemoteHub(cfg), log.StandardLogger()) if err != nil { return err } @@ -71,7 +75,7 @@ func (cli cliItem) install(args []string, downloadOnly bool, force bool, ignoreE if item == nil { msg := suggestNearestMessage(hub, cli.name, name) if !ignoreError { - return fmt.Errorf(msg) + return errors.New(msg) } log.Errorf(msg) @@ -107,10 +111,10 @@ func (cli cliItem) newInstallCmd() *cobra.Command { Example: cli.installHelp.example, Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + ValidArgsFunction: func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compAllItems(cli.name, args, toComplete) }, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, args []string) error { return cli.install(args, downloadOnly, force, ignoreError) }, } @@ -137,15 +141,15 @@ func istalledParentNames(item *cwhub.Item) []string { } func (cli cliItem) remove(args []string, purge bool, force bool, all bool) error { - hub, err := require.Hub(csConfig, nil, log.StandardLogger()) + hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger()) if err != nil { return err } if all { - getter := hub.GetInstalledItems + getter := hub.GetInstalledItemsByType if purge { - getter = hub.GetAllItems + getter = hub.GetItemsByType } items, err := getter(cli.name) @@ -163,6 +167,7 @@ func (cli cliItem) remove(args []string, purge bool, force bool, all bool) error if didRemove { log.Infof("Removed %s", item.Name) + removed++ } } @@ -204,6 +209,7 @@ func (cli cliItem) remove(args []string, purge bool, force bool, all bool) error if didRemove { log.Infof("Removed %s", item.Name) + removed++ } } @@ -231,10 +237,10 @@ func (cli cliItem) newRemoveCmd() *cobra.Command { Example: cli.removeHelp.example, Aliases: []string{"delete"}, DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + ValidArgsFunction: func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cli.name, args, toComplete) }, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, args []string) error { return cli.remove(args, purge, force, all) }, } @@ -248,13 +254,15 @@ func (cli cliItem) newRemoveCmd() *cobra.Command { } func (cli cliItem) upgrade(args []string, force bool, all bool) error { - hub, err := require.Hub(csConfig, require.RemoteHub(csConfig), log.StandardLogger()) + cfg := cli.cfg() + + hub, err := require.Hub(cfg, require.RemoteHub(cfg), log.StandardLogger()) if err != nil { return err } if all { - items, err := hub.GetInstalledItems(cli.name) + items, err := hub.GetInstalledItemsByType(cli.name) if err != nil { return err } @@ -300,6 +308,7 @@ func (cli cliItem) upgrade(args []string, force bool, all bool) error { if didUpdate { log.Infof("Updated %s", item.Name) + updated++ } } @@ -323,10 +332,10 @@ func (cli cliItem) newUpgradeCmd() *cobra.Command { Long: coalesce.String(cli.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", cli.name)), Example: cli.upgradeHelp.example, DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + ValidArgsFunction: func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cli.name, args, toComplete) }, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, args []string) error { return cli.upgrade(args, force, all) }, } @@ -339,21 +348,23 @@ func (cli cliItem) newUpgradeCmd() *cobra.Command { } func (cli cliItem) inspect(args []string, url string, diff bool, rev bool, noMetrics bool) error { + cfg := cli.cfg() + if rev && !diff { - return fmt.Errorf("--rev can only be used with --diff") + return errors.New("--rev can only be used with --diff") } if url != "" { - csConfig.Cscli.PrometheusUrl = url + cfg.Cscli.PrometheusUrl = url } remote := (*cwhub.RemoteHubCfg)(nil) if diff { - remote = require.RemoteHub(csConfig) + remote = require.RemoteHub(cfg) } - hub, err := require.Hub(csConfig, remote, log.StandardLogger()) + hub, err := require.Hub(cfg, remote, log.StandardLogger()) if err != nil { return err } @@ -399,10 +410,10 @@ func (cli cliItem) newInspectCmd() *cobra.Command { Example: cli.inspectHelp.example, Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + ValidArgsFunction: func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cli.name, args, toComplete) }, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, args []string) error { return cli.inspect(args, url, diff, rev, noMetrics) }, } @@ -417,7 +428,7 @@ func (cli cliItem) newInspectCmd() *cobra.Command { } func (cli cliItem) list(args []string, all bool) error { - hub, err := require.Hub(csConfig, nil, log.StandardLogger()) + hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger()) if err != nil { return err } @@ -526,6 +537,7 @@ func (cli cliItem) whyTainted(hub *cwhub.Hub, item *cwhub.Item, reverse bool) st // hack: avoid message "item is tainted by itself" continue } + ret = append(ret, fmt.Sprintf("# %s is tainted by %s", sub.FQName(), taintList)) } } diff --git a/cmd/crowdsec-cli/items.go b/cmd/crowdsec-cli/items.go index ea6d8a256..4ae643151 100644 --- a/cmd/crowdsec-cli/items.go +++ b/cmd/crowdsec-cli/items.go @@ -17,7 +17,7 @@ import ( // selectItems returns a slice of items of a given type, selected by name and sorted by case-insensitive name func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly bool) ([]*cwhub.Item, error) { - itemNames := hub.GetItemNames(itemType) + itemNames := hub.GetNamesByType(itemType) notExist := []string{} @@ -116,7 +116,7 @@ func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item } if err := csvwriter.Write(header); err != nil { - return fmt.Errorf("failed to write header: %s", err) + return fmt.Errorf("failed to write header: %w", err) } for _, itemType := range itemTypes { @@ -132,7 +132,7 @@ func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item } if err := csvwriter.Write(row); err != nil { - return fmt.Errorf("failed to write raw output: %s", err) + return fmt.Errorf("failed to write raw output: %w", err) } } } @@ -150,12 +150,12 @@ func inspectItem(item *cwhub.Item, showMetrics bool) error { enc.SetIndent(2) if err := enc.Encode(item); err != nil { - return fmt.Errorf("unable to encode item: %s", err) + return fmt.Errorf("unable to encode item: %w", err) } case "json": b, err := json.MarshalIndent(*item, "", " ") if err != nil { - return fmt.Errorf("unable to marshal item: %s", err) + return fmt.Errorf("unable to marshal item: %w", err) } fmt.Print(string(b)) diff --git a/cmd/crowdsec-cli/lapi.go b/cmd/crowdsec-cli/lapi.go index 13a9d8d7e..99c3ee45a 100644 --- a/cmd/crowdsec-cli/lapi.go +++ b/cmd/crowdsec-cli/lapi.go @@ -56,7 +56,7 @@ func (cli *cliLapi) status() error { return err } - scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledNamesByType(cwhub.SCENARIOS) if err != nil { return fmt.Errorf("failed to get scenarios: %w", err) } @@ -116,7 +116,6 @@ func (cli *cliLapi) register(apiURL string, outputFile string, machine string) e URL: apiurl, VersionPrefix: LAPIURLPrefix, }, nil) - if err != nil { return fmt.Errorf("api client register: %w", err) } @@ -585,7 +584,7 @@ func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string { } func detectSubNode(node parser.Node, parserCTX parser.UnixParserCtx) []string { - var ret = make([]string, 0) + ret := make([]string, 0) for _, subnode := range node.LeavesNodes { if subnode.Grok.RunTimeRegexp != nil { diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index 446901e4a..0705faa40 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -1,7 +1,9 @@ package main import ( + "fmt" "os" + "path/filepath" "slices" "time" @@ -10,14 +12,18 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/crowdsecurity/go-cs-lib/trace" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/fflag" ) -var ConfigFilePath string -var csConfig *csconfig.Config -var dbClient *database.Client +var ( + ConfigFilePath string + csConfig *csconfig.Config + dbClient *database.Client +) type configGetter func() *csconfig.Config @@ -82,6 +88,11 @@ func loadConfigFor(command string) (*csconfig.Config, string, error) { return nil, "", err } + // set up directory for trace files + if err := trace.Init(filepath.Join(config.ConfigPaths.DataDir, "trace")); err != nil { + return nil, "", fmt.Errorf("while setting up trace directory: %w", err) + } + return config, merged, nil } @@ -249,13 +260,13 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall cmd.AddCommand(NewCLINotifications(cli.cfg).NewCommand()) cmd.AddCommand(NewCLISupport().NewCommand()) cmd.AddCommand(NewCLIPapi(cli.cfg).NewCommand()) - cmd.AddCommand(NewCLICollection().NewCommand()) - cmd.AddCommand(NewCLIParser().NewCommand()) - cmd.AddCommand(NewCLIScenario().NewCommand()) - cmd.AddCommand(NewCLIPostOverflow().NewCommand()) - cmd.AddCommand(NewCLIContext().NewCommand()) - cmd.AddCommand(NewCLIAppsecConfig().NewCommand()) - cmd.AddCommand(NewCLIAppsecRule().NewCommand()) + cmd.AddCommand(NewCLICollection(cli.cfg).NewCommand()) + cmd.AddCommand(NewCLIParser(cli.cfg).NewCommand()) + cmd.AddCommand(NewCLIScenario(cli.cfg).NewCommand()) + cmd.AddCommand(NewCLIPostOverflow(cli.cfg).NewCommand()) + cmd.AddCommand(NewCLIContext(cli.cfg).NewCommand()) + cmd.AddCommand(NewCLIAppsecConfig(cli.cfg).NewCommand()) + cmd.AddCommand(NewCLIAppsecRule(cli.cfg).NewCommand()) if fflag.CscliSetup.IsEnabled() { cmd.AddCommand(NewSetupCmd()) diff --git a/cmd/crowdsec-cli/notifications.go b/cmd/crowdsec-cli/notifications.go index f12333a39..0b5ee5378 100644 --- a/cmd/crowdsec-cli/notifications.go +++ b/cmd/crowdsec-cli/notifications.go @@ -4,6 +4,7 @@ import ( "context" "encoding/csv" "encoding/json" + "errors" "fmt" "io/fs" "net/url" @@ -88,7 +89,7 @@ func (cli *cliNotifications) getPluginConfigs() (map[string]csplugin.PluginConfi return fmt.Errorf("error while traversing directory %s: %w", path, err) } - name := filepath.Join(cfg.ConfigPaths.NotificationDir, info.Name()) //Avoid calling info.Name() twice + name := filepath.Join(cfg.ConfigPaths.NotificationDir, info.Name()) // Avoid calling info.Name() twice if (strings.HasSuffix(name, "yaml") || strings.HasSuffix(name, "yml")) && !(info.IsDir()) { ts, err := csplugin.ParsePluginConfigFile(name) if err != nil { @@ -266,7 +267,7 @@ func (cli *cliNotifications) NewTestCmd() *cobra.Command { if !ok { return fmt.Errorf("plugin name: '%s' does not exist", args[0]) } - //Create a single profile with plugin name as notification name + // Create a single profile with plugin name as notification name return pluginBroker.Init(cfg.PluginConfig, []*csconfig.ProfileCfg{ { Notifications: []string{ @@ -320,8 +321,8 @@ func (cli *cliNotifications) NewTestCmd() *cobra.Command { Alert: alert, } - //time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent - pluginTomb.Kill(fmt.Errorf("terminating")) + // time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent + pluginTomb.Kill(errors.New("terminating")) pluginTomb.Wait() return nil @@ -416,8 +417,8 @@ cscli notifications reinject -a '{"remediation": true,"scenario":"not break } } - //time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent - pluginTomb.Kill(fmt.Errorf("terminating")) + // time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent + pluginTomb.Kill(errors.New("terminating")) pluginTomb.Wait() return nil diff --git a/cmd/crowdsec-cli/papi.go b/cmd/crowdsec-cli/papi.go index e18af94d4..5808fcce5 100644 --- a/cmd/crowdsec-cli/papi.go +++ b/cmd/crowdsec-cli/papi.go @@ -64,25 +64,22 @@ func (cli *cliPapi) NewStatusCmd() *cobra.Command { cfg := cli.cfg() dbClient, err = database.NewClient(cfg.DbConfig) if err != nil { - return fmt.Errorf("unable to initialize database client: %s", err) + return fmt.Errorf("unable to initialize database client: %w", err) } apic, err := apiserver.NewAPIC(cfg.API.Server.OnlineClient, dbClient, cfg.API.Server.ConsoleConfig, cfg.API.Server.CapiWhitelists) - if err != nil { - return fmt.Errorf("unable to initialize API client: %s", err) + return fmt.Errorf("unable to initialize API client: %w", err) } papi, err := apiserver.NewPAPI(apic, dbClient, cfg.API.Server.ConsoleConfig, log.GetLevel()) - if err != nil { - return fmt.Errorf("unable to initialize PAPI client: %s", err) + return fmt.Errorf("unable to initialize PAPI client: %w", err) } perms, err := papi.GetPermissions() - if err != nil { - return fmt.Errorf("unable to get PAPI permissions: %s", err) + return fmt.Errorf("unable to get PAPI permissions: %w", err) } var lastTimestampStr *string lastTimestampStr, err = dbClient.GetConfigItem(apiserver.PapiPullKey) @@ -118,27 +115,26 @@ func (cli *cliPapi) NewSyncCmd() *cobra.Command { dbClient, err = database.NewClient(cfg.DbConfig) if err != nil { - return fmt.Errorf("unable to initialize database client: %s", err) + return fmt.Errorf("unable to initialize database client: %w", err) } apic, err := apiserver.NewAPIC(cfg.API.Server.OnlineClient, dbClient, cfg.API.Server.ConsoleConfig, cfg.API.Server.CapiWhitelists) if err != nil { - return fmt.Errorf("unable to initialize API client: %s", err) + return fmt.Errorf("unable to initialize API client: %w", err) } t.Go(apic.Push) papi, err := apiserver.NewPAPI(apic, dbClient, cfg.API.Server.ConsoleConfig, log.GetLevel()) if err != nil { - return fmt.Errorf("unable to initialize PAPI client: %s", err) + return fmt.Errorf("unable to initialize PAPI client: %w", err) } t.Go(papi.SyncDecisions) err = papi.PullOnce(time.Time{}, true) - if err != nil { - return fmt.Errorf("unable to sync decisions: %s", err) + return fmt.Errorf("unable to sync decisions: %w", err) } log.Infof("Sending acknowledgements to CAPI") diff --git a/cmd/crowdsec-cli/require/require.go b/cmd/crowdsec-cli/require/require.go index 0f5ce182d..708b2d1c7 100644 --- a/cmd/crowdsec-cli/require/require.go +++ b/cmd/crowdsec-cli/require/require.go @@ -1,6 +1,7 @@ package require import ( + "errors" "fmt" "io" @@ -16,7 +17,7 @@ func LAPI(c *csconfig.Config) error { } if c.DisableAPI { - return fmt.Errorf("local API is disabled -- this command must be run on the local API machine") + return errors.New("local API is disabled -- this command must be run on the local API machine") } return nil @@ -32,7 +33,7 @@ func CAPI(c *csconfig.Config) error { func PAPI(c *csconfig.Config) error { if c.API.Server.OnlineClient.Credentials.PapiURL == "" { - return fmt.Errorf("no PAPI URL in configuration") + return errors.New("no PAPI URL in configuration") } return nil @@ -40,7 +41,7 @@ func PAPI(c *csconfig.Config) error { func CAPIRegistered(c *csconfig.Config) error { if c.API.Server.OnlineClient.Credentials == nil { - return fmt.Errorf("the Central API (CAPI) must be configured with 'cscli capi register'") + return errors.New("the Central API (CAPI) must be configured with 'cscli capi register'") } return nil @@ -56,7 +57,7 @@ func DB(c *csconfig.Config) error { func Notifications(c *csconfig.Config) error { if c.ConfigPaths.NotificationDir == "" { - return fmt.Errorf("config_paths.notification_dir is not set in crowdsec config") + return errors.New("config_paths.notification_dir is not set in crowdsec config") } return nil @@ -82,7 +83,7 @@ func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg, logger *logrus.Logger) local := c.Hub if local == nil { - return nil, fmt.Errorf("you must configure cli before interacting with hub") + return nil, errors.New("you must configure cli before interacting with hub") } if logger == nil { diff --git a/cmd/crowdsec-cli/setup.go b/cmd/crowdsec-cli/setup.go index 48dcee089..ba3670848 100644 --- a/cmd/crowdsec-cli/setup.go +++ b/cmd/crowdsec-cli/setup.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "errors" "fmt" "os" "os/exec" @@ -118,9 +119,11 @@ func runSetupDetect(cmd *cobra.Command, args []string) error { switch detectConfigFile { case "-": log.Tracef("Reading detection rules from stdin") + detectReader = os.Stdin default: log.Tracef("Reading detection rules: %s", detectConfigFile) + detectReader, err = os.Open(detectConfigFile) if err != nil { return err @@ -171,6 +174,7 @@ func runSetupDetect(cmd *cobra.Command, args []string) error { _, err := exec.LookPath("systemctl") if err != nil { log.Debug("systemctl not available: snubbing systemd") + snubSystemd = true } } @@ -182,6 +186,7 @@ func runSetupDetect(cmd *cobra.Command, args []string) error { if forcedOSFamily == "" && forcedOSID != "" { log.Debug("force-os-id is set: force-os-family defaults to 'linux'") + forcedOSFamily = "linux" } @@ -219,6 +224,7 @@ func runSetupDetect(cmd *cobra.Command, args []string) error { if err != nil { return err } + fmt.Println(setup) return nil @@ -318,6 +324,7 @@ func runSetupInstallHub(cmd *cobra.Command, args []string) error { func runSetupValidate(cmd *cobra.Command, args []string) error { fromFile := args[0] + input, err := os.ReadFile(fromFile) if err != nil { return fmt.Errorf("while reading stdin: %w", err) @@ -325,7 +332,7 @@ func runSetupValidate(cmd *cobra.Command, args []string) error { if err = setup.Validate(input); err != nil { fmt.Printf("%v\n", err) - return fmt.Errorf("invalid setup file") + return errors.New("invalid setup file") } return nil diff --git a/cmd/crowdsec-cli/simulation.go b/cmd/crowdsec-cli/simulation.go index 6ccac7617..f3c1a6273 100644 --- a/cmd/crowdsec-cli/simulation.go +++ b/cmd/crowdsec-cli/simulation.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "os" "slices" @@ -36,7 +37,7 @@ cscli simulation disable crowdsecurity/ssh-bf`, return err } if cli.cfg().Cscli.SimulationConfig == nil { - return fmt.Errorf("no simulation configured") + return errors.New("no simulation configured") } return nil @@ -99,11 +100,11 @@ func (cli *cliSimulation) NewEnableCmd() *cobra.Command { log.Printf("simulation mode for '%s' enabled", scenario) } if err := cli.dumpSimulationFile(); err != nil { - return fmt.Errorf("simulation enable: %s", err) + return fmt.Errorf("simulation enable: %w", err) } } else if forceGlobalSimulation { if err := cli.enableGlobalSimulation(); err != nil { - return fmt.Errorf("unable to enable global simulation mode: %s", err) + return fmt.Errorf("unable to enable global simulation mode: %w", err) } } else { printHelp(cmd) @@ -146,11 +147,11 @@ func (cli *cliSimulation) NewDisableCmd() *cobra.Command { log.Printf("simulation mode for '%s' disabled", scenario) } if err := cli.dumpSimulationFile(); err != nil { - return fmt.Errorf("simulation disable: %s", err) + return fmt.Errorf("simulation disable: %w", err) } } else if forceGlobalSimulation { if err := cli.disableGlobalSimulation(); err != nil { - return fmt.Errorf("unable to disable global simulation mode: %s", err) + return fmt.Errorf("unable to disable global simulation mode: %w", err) } } else { printHelp(cmd) @@ -202,7 +203,7 @@ func (cli *cliSimulation) enableGlobalSimulation() error { cfg.Cscli.SimulationConfig.Exclusions = []string{} if err := cli.dumpSimulationFile(); err != nil { - return fmt.Errorf("unable to dump simulation file: %s", err) + return fmt.Errorf("unable to dump simulation file: %w", err) } log.Printf("global simulation: enabled") @@ -215,12 +216,12 @@ func (cli *cliSimulation) dumpSimulationFile() error { newConfigSim, err := yaml.Marshal(cfg.Cscli.SimulationConfig) if err != nil { - return fmt.Errorf("unable to marshal simulation configuration: %s", err) + return fmt.Errorf("unable to marshal simulation configuration: %w", err) } err = os.WriteFile(cfg.ConfigPaths.SimulationFilePath, newConfigSim, 0o644) if err != nil { - return fmt.Errorf("write simulation config in '%s' failed: %s", cfg.ConfigPaths.SimulationFilePath, err) + return fmt.Errorf("write simulation config in '%s' failed: %w", cfg.ConfigPaths.SimulationFilePath, err) } log.Debugf("updated simulation file %s", cfg.ConfigPaths.SimulationFilePath) @@ -237,12 +238,12 @@ func (cli *cliSimulation) disableGlobalSimulation() error { newConfigSim, err := yaml.Marshal(cfg.Cscli.SimulationConfig) if err != nil { - return fmt.Errorf("unable to marshal new simulation configuration: %s", err) + return fmt.Errorf("unable to marshal new simulation configuration: %w", err) } err = os.WriteFile(cfg.ConfigPaths.SimulationFilePath, newConfigSim, 0o644) if err != nil { - return fmt.Errorf("unable to write new simulation config in '%s': %s", cfg.ConfigPaths.SimulationFilePath, err) + return fmt.Errorf("unable to write new simulation config in '%s': %w", cfg.ConfigPaths.SimulationFilePath, err) } log.Printf("global simulation: disabled") @@ -269,8 +270,10 @@ func (cli *cliSimulation) status() { } } else { log.Println("global simulation: disabled") + if len(cfg.Cscli.SimulationConfig.Exclusions) > 0 { log.Println("Scenarios in simulation mode :") + for _, scenario := range cfg.Cscli.SimulationConfig.Exclusions { log.Printf(" - %s", scenario) } diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go index 661950fa8..a48edeeeb 100644 --- a/cmd/crowdsec-cli/support.go +++ b/cmd/crowdsec-cli/support.go @@ -4,6 +4,7 @@ import ( "archive/zip" "bytes" "context" + "errors" "fmt" "io" "net/http" @@ -12,12 +13,14 @@ import ( "path/filepath" "regexp" "strings" + "time" "github.com/blackfireio/osinfo" "github.com/go-openapi/strfmt" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/go-cs-lib/version" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" @@ -47,6 +50,7 @@ const ( SUPPORT_CAPI_STATUS_PATH = "capi_status.txt" SUPPORT_ACQUISITION_CONFIG_BASE_PATH = "config/acquis/" SUPPORT_CROWDSEC_PROFILE_PATH = "config/profiles.yaml" + SUPPORT_CRASH_PATH = "crash/" ) // from https://github.com/acarl005/stripansi @@ -62,7 +66,7 @@ func collectMetrics() ([]byte, []byte, error) { if csConfig.Cscli.PrometheusUrl == "" { log.Warn("No Prometheus URL configured, metrics will not be collected") - return nil, nil, fmt.Errorf("prometheus_uri is not set") + return nil, nil, errors.New("prometheus_uri is not set") } humanMetrics := bytes.NewBuffer(nil) @@ -70,7 +74,7 @@ func collectMetrics() ([]byte, []byte, error) { ms := NewMetricStore() if err := ms.Fetch(csConfig.Cscli.PrometheusUrl); err != nil { - return nil, nil, fmt.Errorf("could not fetch prometheus metrics: %s", err) + return nil, nil, fmt.Errorf("could not fetch prometheus metrics: %w", err) } if err := ms.Format(humanMetrics, nil, "human", false); err != nil { @@ -79,21 +83,21 @@ func collectMetrics() ([]byte, []byte, error) { req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl, nil) if err != nil { - return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %s", err) + return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %w", err) } client := &http.Client{} resp, err := client.Do(req) if err != nil { - return nil, nil, fmt.Errorf("could not get metrics from prometheus endpoint: %s", err) + return nil, nil, fmt.Errorf("could not get metrics from prometheus endpoint: %w", err) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { - return nil, nil, fmt.Errorf("could not read metrics from prometheus endpoint: %s", err) + return nil, nil, fmt.Errorf("could not read metrics from prometheus endpoint: %w", err) } return humanMetrics.Bytes(), body, nil @@ -121,19 +125,18 @@ func collectOSInfo() ([]byte, error) { log.Info("Collecting OS info") info, err := osinfo.GetOSInfo() - if err != nil { return nil, err } w := bytes.NewBuffer(nil) - w.WriteString(fmt.Sprintf("Architecture: %s\n", info.Architecture)) - w.WriteString(fmt.Sprintf("Family: %s\n", info.Family)) - w.WriteString(fmt.Sprintf("ID: %s\n", info.ID)) - w.WriteString(fmt.Sprintf("Name: %s\n", info.Name)) - w.WriteString(fmt.Sprintf("Codename: %s\n", info.Codename)) - w.WriteString(fmt.Sprintf("Version: %s\n", info.Version)) - w.WriteString(fmt.Sprintf("Build: %s\n", info.Build)) + fmt.Fprintf(w, "Architecture: %s\n", info.Architecture) + fmt.Fprintf(w, "Family: %s\n", info.Family) + fmt.Fprintf(w, "ID: %s\n", info.ID) + fmt.Fprintf(w, "Name: %s\n", info.Name) + fmt.Fprintf(w, "Codename: %s\n", info.Codename) + fmt.Fprintf(w, "Version: %s\n", info.Version) + fmt.Fprintf(w, "Build: %s\n", info.Build) return w.Bytes(), nil } @@ -163,7 +166,7 @@ func collectBouncers(dbClient *database.Client) ([]byte, error) { bouncers, err := dbClient.ListBouncers() if err != nil { - return nil, fmt.Errorf("unable to list bouncers: %s", err) + return nil, fmt.Errorf("unable to list bouncers: %w", err) } getBouncersTable(out, bouncers) @@ -176,7 +179,7 @@ func collectAgents(dbClient *database.Client) ([]byte, error) { machines, err := dbClient.ListMachines() if err != nil { - return nil, fmt.Errorf("unable to list machines: %s", err) + return nil, fmt.Errorf("unable to list machines: %w", err) } getAgentsTable(out, machines) @@ -196,7 +199,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str return []byte(fmt.Sprintf("cannot parse API URL: %s", err)) } - scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledNamesByType(cwhub.SCENARIOS) if err != nil { return []byte(fmt.Sprintf("could not collect scenarios: %s", err)) } @@ -264,6 +267,11 @@ func collectAcquisitionConfig() map[string][]byte { return ret } +func collectCrash() ([]string, error) { + log.Info("Collecting crash dumps") + return trace.List() +} + type cliSupport struct{} func NewCLISupport() *cliSupport { @@ -431,11 +439,31 @@ cscli support dump -f /tmp/crowdsec-support.zip } } + crash, err := collectCrash() + if err != nil { + log.Errorf("could not collect crash dumps: %s", err) + } + + for _, filename := range crash { + content, err := os.ReadFile(filename) + if err != nil { + log.Errorf("could not read crash dump %s: %s", filename, err) + } + + infos[SUPPORT_CRASH_PATH+filepath.Base(filename)] = content + } + w := bytes.NewBuffer(nil) zipWriter := zip.NewWriter(w) for filename, data := range infos { - fw, err := zipWriter.Create(filename) + header := &zip.FileHeader{ + Name: filename, + Method: zip.Deflate, + // TODO: retain mtime where possible (esp. trace) + Modified: time.Now(), + } + fw, err := zipWriter.CreateHeader(header) if err != nil { log.Errorf("Could not add zip entry for %s: %s", filename, err) continue diff --git a/cmd/crowdsec/lapiclient.go b/cmd/crowdsec/lapiclient.go index fd29aa9d9..f12aea5ac 100644 --- a/cmd/crowdsec/lapiclient.go +++ b/cmd/crowdsec/lapiclient.go @@ -17,12 +17,12 @@ import ( ) func AuthenticatedLAPIClient(credentials csconfig.ApiCredentialsCfg, hub *cwhub.Hub) (*apiclient.ApiClient, error) { - scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledNamesByType(cwhub.SCENARIOS) if err != nil { return nil, fmt.Errorf("loading list of installed hub scenarios: %w", err) } - appsecRules, err := hub.GetInstalledItemNames(cwhub.APPSEC_RULES) + appsecRules, err := hub.GetInstalledNamesByType(cwhub.APPSEC_RULES) if err != nil { return nil, fmt.Errorf("loading list of installed hub appsec rules: %w", err) } @@ -52,11 +52,11 @@ func AuthenticatedLAPIClient(credentials csconfig.ApiCredentialsCfg, hub *cwhub. PapiURL: papiURL, VersionPrefix: "v1", UpdateScenario: func() ([]string, error) { - scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledNamesByType(cwhub.SCENARIOS) if err != nil { return nil, err } - appsecRules, err := hub.GetInstalledItemNames(cwhub.APPSEC_RULES) + appsecRules, err := hub.GetInstalledNamesByType(cwhub.APPSEC_RULES) if err != nil { return nil, err } diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index 5f04e9b99..0d96692ba 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -6,6 +6,7 @@ import ( "fmt" _ "net/http/pprof" "os" + "path/filepath" "runtime" "runtime/pprof" "strings" @@ -14,6 +15,8 @@ import ( log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" + "github.com/crowdsecurity/go-cs-lib/trace" + "github.com/crowdsecurity/crowdsec/pkg/acquisition" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csplugin" @@ -96,8 +99,8 @@ func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error { buckets = leakybucket.NewBuckets() log.Infof("Loading %d scenario files", len(files)) - holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, hub, files, &bucketsTomb, buckets, flags.OrderEvent) + holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, hub, files, &bucketsTomb, buckets, flags.OrderEvent) if err != nil { return fmt.Errorf("scenario loading failed: %w", err) } @@ -230,6 +233,10 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo return nil, fmt.Errorf("while loading configuration file: %w", err) } + if err := trace.Init(filepath.Join(cConfig.ConfigPaths.DataDir, "trace")); err != nil { + return nil, fmt.Errorf("while setting up trace directory: %w", err) + } + cConfig.Common.LogLevel = newLogLevel(cConfig.Common.LogLevel, flags) if dumpFolder != "" { diff --git a/cmd/crowdsec/metrics.go b/cmd/crowdsec/metrics.go index d670051ce..cc0c118b4 100644 --- a/cmd/crowdsec/metrics.go +++ b/cmd/crowdsec/metrics.go @@ -3,7 +3,6 @@ package main import ( "fmt" "net/http" - "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -22,7 +21,8 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/parser" ) -/*prometheus*/ +// Prometheus + var globalParserHits = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "cs_parser_hits_total", @@ -30,6 +30,7 @@ var globalParserHits = prometheus.NewCounterVec( }, []string{"source", "type"}, ) + var globalParserHitsOk = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "cs_parser_hits_ok_total", @@ -37,6 +38,7 @@ var globalParserHitsOk = prometheus.NewCounterVec( }, []string{"source", "type"}, ) + var globalParserHitsKo = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "cs_parser_hits_ko_total", @@ -116,9 +118,7 @@ func computeDynamicMetrics(next http.Handler, dbClient *database.Client) http.Ha return } - decisionsFilters := make(map[string][]string, 0) - - decisions, err := dbClient.QueryDecisionCountByScenario(decisionsFilters) + decisions, err := dbClient.QueryDecisionCountByScenario() if err != nil { log.Errorf("Error querying decisions for metrics: %v", err) next.ServeHTTP(w, r) @@ -139,7 +139,6 @@ func computeDynamicMetrics(next http.Handler, dbClient *database.Client) http.Ha } alerts, err := dbClient.AlertsCountPerScenario(alertsFilter) - if err != nil { log.Errorf("Error querying alerts for metrics: %v", err) next.ServeHTTP(w, r) @@ -194,7 +193,6 @@ func servePrometheus(config *csconfig.PrometheusCfg, dbClient *database.Client, defer trace.CatchPanic("crowdsec/servePrometheus") http.Handle("/metrics", computeDynamicMetrics(promhttp.Handler(), dbClient)) - log.Debugf("serving metrics after %s ms", time.Since(crowdsecT0)) if err := http.ListenAndServe(fmt.Sprintf("%s:%d", config.ListenAddr, config.ListenPort), nil); err != nil { // in time machine, we most likely have the LAPI using the port diff --git a/cmd/crowdsec/serve.go b/cmd/crowdsec/serve.go index c8ccd4d5d..9da3d8010 100644 --- a/cmd/crowdsec/serve.go +++ b/cmd/crowdsec/serve.go @@ -391,7 +391,7 @@ func Serve(cConfig *csconfig.Config, agentReady chan bool) error { } if cConfig.Common != nil && cConfig.Common.Daemonize { - csdaemon.NotifySystemd(log.StandardLogger()) + csdaemon.Notify(csdaemon.Ready, log.StandardLogger()) // wait for signals return HandleSignals(cConfig) } diff --git a/go.mod b/go.mod index 04f34e648..70d819a40 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/corazawaf/libinjection-go v0.1.2 github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 - github.com/crowdsecurity/go-cs-lib v0.0.6 + github.com/crowdsecurity/go-cs-lib v0.0.10 github.com/crowdsecurity/grokky v0.2.1 github.com/crowdsecurity/machineid v1.0.2 github.com/davecgh/go-spew v1.1.1 diff --git a/go.sum b/go.sum index 29e23f02a..750439e4f 100644 --- a/go.sum +++ b/go.sum @@ -102,8 +102,8 @@ github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607 h1:hyrYw3h github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607/go.mod h1:br36fEqurGYZQGit+iDYsIzW0FF6VufMbDzyyLxEuPA= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= -github.com/crowdsecurity/go-cs-lib v0.0.6 h1:Ef6MylXe0GaJE9vrfvxEdbHb31+JUP1os+murPz7Pos= -github.com/crowdsecurity/go-cs-lib v0.0.6/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k= +github.com/crowdsecurity/go-cs-lib v0.0.10 h1:Twt/y/rYCUspGY1zxDnGurL2svRSREAz+2+puLepd9c= +github.com/crowdsecurity/go-cs-lib v0.0.10/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k= github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4= github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM= github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc= diff --git a/pkg/alertcontext/config.go b/pkg/alertcontext/config.go index 74ca1523a..21d16db39 100644 --- a/pkg/alertcontext/config.go +++ b/pkg/alertcontext/config.go @@ -104,7 +104,7 @@ func LoadConsoleContext(c *csconfig.Config, hub *cwhub.Hub) error { c.Crowdsec.ContextToSend = make(map[string][]string, 0) if hub != nil { - items, err := hub.GetInstalledItems(cwhub.CONTEXTS) + items, err := hub.GetInstalledItemsByType(cwhub.CONTEXTS) if err != nil { return err } diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 7989cfc1d..6592c8bbf 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -84,11 +84,16 @@ func recoverFromPanic(c *gin.Context) { } if brokenPipe { - log.Warningf("client %s disconnected : %s", c.ClientIP(), err) + log.Warningf("client %s disconnected: %s", c.ClientIP(), err) c.Abort() } else { - filename := trace.WriteStackTrace(err) - log.Warningf("client %s error : %s", c.ClientIP(), err) + log.Warningf("client %s error: %s", c.ClientIP(), err) + + filename, err := trace.WriteStackTrace(err) + if err != nil { + log.Errorf("also while writing stacktrace: %s", err) + } + log.Warningf("stacktrace written to %s, please join to your issue", filename) c.AbortWithStatus(http.StatusInternalServerError) } diff --git a/pkg/csconfig/database.go b/pkg/csconfig/database.go index a7bc57eef..9a9ed9a9f 100644 --- a/pkg/csconfig/database.go +++ b/pkg/csconfig/database.go @@ -76,26 +76,24 @@ func (c *Config) LoadDBConfig(inCli bool) error { if c.DbConfig.UseWal == nil { dbDir := filepath.Dir(c.DbConfig.DbPath) isNetwork, fsType, err := types.IsNetworkFS(dbDir) - if err != nil { + switch { + case err != nil: log.Warnf("unable to determine if database is on network filesystem: %s", err) log.Warning("You are using sqlite without WAL, this can have a performance impact. If you do not store the database in a network share, set db_config.use_wal to true. Set explicitly to false to disable this warning.") - return nil - } - if isNetwork { + case isNetwork: log.Debugf("database is on network filesystem (%s), setting useWal to false", fsType) c.DbConfig.UseWal = ptr.Of(false) - } else { + default: log.Debugf("database is on local filesystem (%s), setting useWal to true", fsType) c.DbConfig.UseWal = ptr.Of(true) } } else if *c.DbConfig.UseWal { dbDir := filepath.Dir(c.DbConfig.DbPath) isNetwork, fsType, err := types.IsNetworkFS(dbDir) - if err != nil { + switch { + case err != nil: log.Warnf("unable to determine if database is on network filesystem: %s", err) - return nil - } - if isNetwork { + case isNetwork: log.Warnf("database seems to be stored on a network share (%s), but useWal is set to true. Proceed at your own risk.", fsType) } } diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 44e24020d..6b9f56b2e 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -214,9 +214,9 @@ func (h *Hub) GetItemFQ(itemFQName string) (*Item, error) { return i, nil } -// GetItemNames returns a slice of (full) item names for a given type +// GetNamesByType returns a slice of (full) item names for a given type // (eg. for collections: crowdsecurity/apache2 crowdsecurity/nginx). -func (h *Hub) GetItemNames(itemType string) []string { +func (h *Hub) GetNamesByType(itemType string) []string { m := h.GetItemMap(itemType) if m == nil { return nil @@ -230,8 +230,8 @@ func (h *Hub) GetItemNames(itemType string) []string { return names } -// GetAllItems returns a slice of all the items of a given type, installed or not. -func (h *Hub) GetAllItems(itemType string) ([]*Item, error) { +// GetItemsByType returns a slice of all the items of a given type, installed or not. +func (h *Hub) GetItemsByType(itemType string) ([]*Item, error) { if !slices.Contains(ItemTypes, itemType) { return nil, fmt.Errorf("invalid item type %s", itemType) } @@ -250,8 +250,8 @@ func (h *Hub) GetAllItems(itemType string) ([]*Item, error) { return ret, nil } -// GetInstalledItems returns a slice of the installed items of a given type. -func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) { +// GetInstalledItemsByType returns a slice of the installed items of a given type. +func (h *Hub) GetInstalledItemsByType(itemType string) ([]*Item, error) { if !slices.Contains(ItemTypes, itemType) { return nil, fmt.Errorf("invalid item type %s", itemType) } @@ -269,9 +269,9 @@ func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) { return retItems, nil } -// GetInstalledItemNames returns the names of the installed items of a given type. -func (h *Hub) GetInstalledItemNames(itemType string) ([]string, error) { - items, err := h.GetInstalledItems(itemType) +// GetInstalledNamesByType returns the names of the installed items of a given type. +func (h *Hub) GetInstalledNamesByType(itemType string) ([]string, error) { + items, err := h.GetInstalledItemsByType(itemType) if err != nil { return nil, err } diff --git a/pkg/database/alerts.go b/pkg/database/alerts.go index 0502c2531..5559cbb39 100644 --- a/pkg/database/alerts.go +++ b/pkg/database/alerts.go @@ -636,14 +636,24 @@ func (c *Client) createAlertChunk(machineID string, owner *ent.Machine, alerts [ if len(alertItem.Meta) > 0 { metaBulk := make([]*ent.MetaCreate, len(alertItem.Meta)) for i, metaItem := range alertItem.Meta { + key := metaItem.Key + value := metaItem.Value + if len(metaItem.Value) > 4095 { + c.Log.Warningf("truncated meta %s : value too long", metaItem.Key) + value = value[:4095] + } + if len(metaItem.Key) > 255 { + c.Log.Warningf("truncated meta %s : key too long", metaItem.Key) + key = key[:255] + } metaBulk[i] = c.Ent.Meta.Create(). - SetKey(metaItem.Key). - SetValue(metaItem.Value) + SetKey(key). + SetValue(value) } metas, err = c.Ent.Meta.CreateBulk(metaBulk...).Save(c.CTX) if err != nil { - return nil, errors.Wrapf(BulkError, "creating alert meta: %s", err) + c.Log.Warningf("error creating alert meta: %s", err) } } diff --git a/pkg/database/decisions.go b/pkg/database/decisions.go index 3175a916f..20a49c791 100644 --- a/pkg/database/decisions.go +++ b/pkg/database/decisions.go @@ -37,6 +37,7 @@ func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string] if v[0] == "false" { query = query.Where(decision.SimulatedEQ(false)) } + delete(filter, "simulated") } else { query = query.Where(decision.SimulatedEQ(false)) @@ -49,7 +50,7 @@ func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string] if err != nil { return nil, errors.Wrapf(InvalidFilter, "invalid contains value : %s", err) } - case "scopes", "scope": //Swagger mentions both of them, let's just support both to make sure we don't break anything + case "scopes", "scope": // Swagger mentions both of them, let's just support both to make sure we don't break anything scopes := strings.Split(value[0], ",") for i, scope := range scopes { switch strings.ToLower(scope) { @@ -63,6 +64,7 @@ func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string] scopes[i] = types.AS } } + query = query.Where(decision.ScopeIn(scopes...)) case "value": query = query.Where(decision.ValueEQ(value[0])) @@ -164,11 +166,11 @@ func (c *Client) QueryExpiredDecisionsWithFilters(filters map[string][]string) ( return data, nil } -func (c *Client) QueryDecisionCountByScenario(filters map[string][]string) ([]*DecisionsByScenario, error) { +func (c *Client) QueryDecisionCountByScenario() ([]*DecisionsByScenario, error) { query := c.Ent.Decision.Query().Where( decision.UntilGT(time.Now().UTC()), ) - query, err := BuildDecisionRequestWithFilter(query, filters) + query, err := BuildDecisionRequestWithFilter(query, make(map[string][]string)) if err != nil { c.Log.Warningf("QueryDecisionCountByScenario : %s", err) @@ -277,10 +279,12 @@ func (c *Client) QueryNewDecisionsSinceWithFilters(since time.Time, filters map[ decision.CreatedAtGT(since), decision.UntilGT(time.Now().UTC()), ) - //Allow a bouncer to ask for non-deduplicated results + + // Allow a bouncer to ask for non-deduplicated results if v, ok := filters["dedup"]; !ok || v[0] != "false" { query = query.Where(longestDecisionForScopeTypeValue) } + query, err := BuildDecisionRequestWithFilter(query, filters) if err != nil { c.Log.Warningf("QueryNewDecisionsSinceWithFilters : %s", err) @@ -294,17 +298,20 @@ func (c *Client) QueryNewDecisionsSinceWithFilters(since time.Time, filters map[ c.Log.Warningf("QueryNewDecisionsSinceWithFilters : %s", err) return []*ent.Decision{}, errors.Wrapf(QueryFail, "new decisions since '%s'", since.String()) } + return data, nil } -func (c *Client) DeleteDecisionById(decisionId int) ([]*ent.Decision, error) { - toDelete, err := c.Ent.Decision.Query().Where(decision.IDEQ(decisionId)).All(c.CTX) +func (c *Client) DeleteDecisionById(decisionID int) ([]*ent.Decision, error) { + toDelete, err := c.Ent.Decision.Query().Where(decision.IDEQ(decisionID)).All(c.CTX) if err != nil { c.Log.Warningf("DeleteDecisionById : %s", err) - return nil, errors.Wrapf(DeleteFail, "decision with id '%d' doesn't exist", decisionId) + return nil, errors.Wrapf(DeleteFail, "decision with id '%d' doesn't exist", decisionID) } + count, err := c.BulkDeleteDecisions(toDelete, false) c.Log.Debugf("deleted %d decisions", count) + return toDelete, err } @@ -317,6 +324,7 @@ func (c *Client) DeleteDecisionsWithFilter(filter map[string][]string) (string, else, return bans that are *contained* by the given value (value is the outer) */ decisions := c.Ent.Decision.Query() + for param, value := range filter { switch param { case "contains": @@ -359,48 +367,48 @@ func (c *Client) DeleteDecisionsWithFilter(filter map[string][]string) (string, } else if ip_sz == 16 { if contains { /*decision contains {start_ip,end_ip}*/ decisions = decisions.Where(decision.And( - //matching addr size + // matching addr size decision.IPSizeEQ(int64(ip_sz)), decision.Or( - //decision.start_ip < query.start_ip + // decision.start_ip < query.start_ip decision.StartIPLT(start_ip), decision.And( - //decision.start_ip == query.start_ip + // decision.start_ip == query.start_ip decision.StartIPEQ(start_ip), - //decision.start_suffix <= query.start_suffix + // decision.start_suffix <= query.start_suffix decision.StartSuffixLTE(start_sfx), )), decision.Or( - //decision.end_ip > query.end_ip + // decision.end_ip > query.end_ip decision.EndIPGT(end_ip), decision.And( - //decision.end_ip == query.end_ip + // decision.end_ip == query.end_ip decision.EndIPEQ(end_ip), - //decision.end_suffix >= query.end_suffix + // decision.end_suffix >= query.end_suffix decision.EndSuffixGTE(end_sfx), ), ), )) } else { decisions = decisions.Where(decision.And( - //matching addr size + // matching addr size decision.IPSizeEQ(int64(ip_sz)), decision.Or( - //decision.start_ip > query.start_ip + // decision.start_ip > query.start_ip decision.StartIPGT(start_ip), decision.And( - //decision.start_ip == query.start_ip + // decision.start_ip == query.start_ip decision.StartIPEQ(start_ip), - //decision.start_suffix >= query.start_suffix + // decision.start_suffix >= query.start_suffix decision.StartSuffixGTE(start_sfx), )), decision.Or( - //decision.end_ip < query.end_ip + // decision.end_ip < query.end_ip decision.EndIPLT(end_ip), decision.And( - //decision.end_ip == query.end_ip + // decision.end_ip == query.end_ip decision.EndIPEQ(end_ip), - //decision.end_suffix <= query.end_suffix + // decision.end_suffix <= query.end_suffix decision.EndSuffixLTE(end_sfx), ), ), @@ -415,11 +423,13 @@ func (c *Client) DeleteDecisionsWithFilter(filter map[string][]string) (string, c.Log.Warningf("DeleteDecisionsWithFilter : %s", err) return "0", nil, errors.Wrap(DeleteFail, "decisions with provided filter") } + count, err := c.BulkDeleteDecisions(toDelete, false) if err != nil { c.Log.Warningf("While deleting decisions : %s", err) return "0", nil, errors.Wrap(DeleteFail, "decisions with provided filter") } + return strconv.Itoa(count), toDelete, nil } @@ -432,6 +442,7 @@ func (c *Client) SoftDeleteDecisionsWithFilter(filter map[string][]string) (stri /*if contains is true, return bans that *contains* the given value (value is the inner) else, return bans that are *contained* by the given value (value is the outer)*/ decisions := c.Ent.Decision.Query().Where(decision.UntilGT(time.Now().UTC())) + for param, value := range filter { switch param { case "contains": @@ -480,24 +491,24 @@ func (c *Client) SoftDeleteDecisionsWithFilter(filter map[string][]string) (stri /*decision contains {start_ip,end_ip}*/ if contains { decisions = decisions.Where(decision.And( - //matching addr size + // matching addr size decision.IPSizeEQ(int64(ip_sz)), decision.Or( - //decision.start_ip < query.start_ip + // decision.start_ip < query.start_ip decision.StartIPLT(start_ip), decision.And( - //decision.start_ip == query.start_ip + // decision.start_ip == query.start_ip decision.StartIPEQ(start_ip), - //decision.start_suffix <= query.start_suffix + // decision.start_suffix <= query.start_suffix decision.StartSuffixLTE(start_sfx), )), decision.Or( - //decision.end_ip > query.end_ip + // decision.end_ip > query.end_ip decision.EndIPGT(end_ip), decision.And( - //decision.end_ip == query.end_ip + // decision.end_ip == query.end_ip decision.EndIPEQ(end_ip), - //decision.end_suffix >= query.end_suffix + // decision.end_suffix >= query.end_suffix decision.EndSuffixGTE(end_sfx), ), ), @@ -505,24 +516,24 @@ func (c *Client) SoftDeleteDecisionsWithFilter(filter map[string][]string) (stri } else { /*decision is contained within {start_ip,end_ip}*/ decisions = decisions.Where(decision.And( - //matching addr size + // matching addr size decision.IPSizeEQ(int64(ip_sz)), decision.Or( - //decision.start_ip > query.start_ip + // decision.start_ip > query.start_ip decision.StartIPGT(start_ip), decision.And( - //decision.start_ip == query.start_ip + // decision.start_ip == query.start_ip decision.StartIPEQ(start_ip), - //decision.start_suffix >= query.start_suffix + // decision.start_suffix >= query.start_suffix decision.StartSuffixGTE(start_sfx), )), decision.Or( - //decision.end_ip < query.end_ip + // decision.end_ip < query.end_ip decision.EndIPLT(end_ip), decision.And( - //decision.end_ip == query.end_ip + // decision.end_ip == query.end_ip decision.EndIPEQ(end_ip), - //decision.end_suffix <= query.end_suffix + // decision.end_suffix <= query.end_suffix decision.EndSuffixLTE(end_sfx), ), ), @@ -531,6 +542,7 @@ func (c *Client) SoftDeleteDecisionsWithFilter(filter map[string][]string) (stri } else if ip_sz != 0 { return "0", nil, errors.Wrapf(InvalidFilter, "Unknown ip size %d", ip_sz) } + DecisionsToDelete, err := decisions.All(c.CTX) if err != nil { c.Log.Warningf("SoftDeleteDecisionsWithFilter : %s", err) @@ -541,13 +553,14 @@ func (c *Client) SoftDeleteDecisionsWithFilter(filter map[string][]string) (stri if err != nil { return "0", nil, errors.Wrapf(DeleteFail, "soft delete decisions with provided filter : %s", err) } + return strconv.Itoa(count), DecisionsToDelete, err } -// BulkDeleteDecisions set the expiration of a bulk of decisions to now() or hard deletes them. +// BulkDeleteDecisions sets the expiration of a bulk of decisions to now() or hard deletes them. // We are doing it this way so we can return impacted decisions for sync with CAPI/PAPI func (c *Client) BulkDeleteDecisions(decisionsToDelete []*ent.Decision, softDelete bool) (int, error) { - const bulkSize = 256 //scientifically proven to be the best value for bulk delete + const bulkSize = 256 // scientifically proven to be the best value for bulk delete var ( nbUpdates int @@ -576,6 +589,7 @@ func (c *Client) BulkDeleteDecisions(decisionsToDelete []*ent.Decision, softDele return totalUpdates, fmt.Errorf("hard delete decisions with provided filter: %w", err) } } + totalUpdates += nbUpdates } @@ -612,6 +626,7 @@ func (c *Client) CountDecisionsByValue(decisionValue string) (int, error) { contains := true decisions := c.Ent.Decision.Query() + decisions, err = applyStartIpEndIpFilter(decisions, contains, ip_sz, start_ip, start_sfx, end_ip, end_sfx) if err != nil { return 0, errors.Wrapf(err, "fail to apply StartIpEndIpFilter") @@ -667,6 +682,7 @@ func applyStartIpEndIpFilter(decisions *ent.DecisionQuery, contains bool, ip_sz decision.IPSizeEQ(int64(ip_sz)), )) } + return decisions, nil } @@ -674,24 +690,24 @@ func applyStartIpEndIpFilter(decisions *ent.DecisionQuery, contains bool, ip_sz /*decision contains {start_ip,end_ip}*/ if contains { decisions = decisions.Where(decision.And( - //matching addr size + // matching addr size decision.IPSizeEQ(int64(ip_sz)), decision.Or( - //decision.start_ip < query.start_ip + // decision.start_ip < query.start_ip decision.StartIPLT(start_ip), decision.And( - //decision.start_ip == query.start_ip + // decision.start_ip == query.start_ip decision.StartIPEQ(start_ip), - //decision.start_suffix <= query.start_suffix + // decision.start_suffix <= query.start_suffix decision.StartSuffixLTE(start_sfx), )), decision.Or( - //decision.end_ip > query.end_ip + // decision.end_ip > query.end_ip decision.EndIPGT(end_ip), decision.And( - //decision.end_ip == query.end_ip + // decision.end_ip == query.end_ip decision.EndIPEQ(end_ip), - //decision.end_suffix >= query.end_suffix + // decision.end_suffix >= query.end_suffix decision.EndSuffixGTE(end_sfx), ), ), @@ -699,29 +715,30 @@ func applyStartIpEndIpFilter(decisions *ent.DecisionQuery, contains bool, ip_sz } else { /*decision is contained within {start_ip,end_ip}*/ decisions = decisions.Where(decision.And( - //matching addr size + // matching addr size decision.IPSizeEQ(int64(ip_sz)), decision.Or( - //decision.start_ip > query.start_ip + // decision.start_ip > query.start_ip decision.StartIPGT(start_ip), decision.And( - //decision.start_ip == query.start_ip + // decision.start_ip == query.start_ip decision.StartIPEQ(start_ip), - //decision.start_suffix >= query.start_suffix + // decision.start_suffix >= query.start_suffix decision.StartSuffixGTE(start_sfx), )), decision.Or( - //decision.end_ip < query.end_ip + // decision.end_ip < query.end_ip decision.EndIPLT(end_ip), decision.And( - //decision.end_ip == query.end_ip + // decision.end_ip == query.end_ip decision.EndIPEQ(end_ip), - //decision.end_suffix <= query.end_suffix + // decision.end_suffix <= query.end_suffix decision.EndSuffixLTE(end_sfx), ), ), )) } + return decisions, nil } @@ -735,8 +752,10 @@ func applyStartIpEndIpFilter(decisions *ent.DecisionQuery, contains bool, ip_sz func decisionPredicatesFromStr(s string, predicateFunc func(string) predicate.Decision) []predicate.Decision { words := strings.Split(s, ",") predicates := make([]predicate.Decision, len(words)) + for i, word := range words { predicates[i] = predicateFunc(word) } + return predicates } diff --git a/pkg/models/localapi_swagger.yaml b/pkg/models/localapi_swagger.yaml index 66132e5e3..d167da9b6 100644 --- a/pkg/models/localapi_swagger.yaml +++ b/pkg/models/localapi_swagger.yaml @@ -26,10 +26,10 @@ produces: paths: /decisions/stream: get: - description: Returns a list of new/expired decisions. Intended for bouncers that need to "stream" decisions + description: Returns a list of new/expired decisions. Intended for remediation component that need to "stream" decisions summary: getDecisionsStream tags: - - bouncers + - Remediation component operationId: getDecisionsStream deprecated: false produces: @@ -39,7 +39,7 @@ paths: in: query required: false type: boolean - description: 'If true, means that the bouncers is starting and a full list must be provided' + description: 'If true, means that the remediation component is starting and a full list must be provided' - name: scopes in: query required: false @@ -73,10 +73,10 @@ paths: security: - APIKeyAuthorizer: [] head: - description: Returns a list of new/expired decisions. Intended for bouncers that need to "stream" decisions + description: Returns a list of new/expired decisions. Intended for remediation component that need to "stream" decisions summary: GetDecisionsStream tags: - - bouncers + - Remediation component operationId: headDecisionsStream deprecated: false produces: @@ -100,7 +100,7 @@ paths: description: Returns information about existing decisions summary: getDecisions tags: - - bouncers + - Remediation component operationId: getDecisions deprecated: false produces: @@ -164,7 +164,7 @@ paths: description: Returns information about existing decisions summary: GetDecisions tags: - - bouncers + - Remediation component operationId: headDecisions deprecated: false produces: @@ -1008,7 +1008,7 @@ definitions: title: "error response" description: "error response return by the API" tags: - - name: bouncers + - name: Remediation component description: 'Operations about decisions : bans, captcha, rate-limit etc.' - name: watchers description: 'Operations about watchers : cscli & crowdsec' diff --git a/pkg/types/getfstype.go b/pkg/types/getfstype.go index 67e018782..25790ecb6 100644 --- a/pkg/types/getfstype.go +++ b/pkg/types/getfstype.go @@ -1,4 +1,4 @@ -//go:build !windows +//go:build !windows && !freebsd package types diff --git a/pkg/types/getfstype_freebsd.go b/pkg/types/getfstype_freebsd.go new file mode 100644 index 000000000..8fbe3dd7c --- /dev/null +++ b/pkg/types/getfstype_freebsd.go @@ -0,0 +1,25 @@ +//go:build freebsd + +package types + +import ( + "fmt" + "syscall" +) + +func GetFSType(path string) (string, error) { + var fsStat syscall.Statfs_t + + if err := syscall.Statfs(path, &fsStat); err != nil { + return "", fmt.Errorf("failed to get filesystem type: %w", err) + } + + bs := fsStat.Fstypename + + b := make([]byte, len(bs)) + for i, v := range bs { + b[i] = byte(v) + } + + return string(b), nil +} diff --git a/test/bats.mk b/test/bats.mk index 0cc5deb9b..8f507cb65 100644 --- a/test/bats.mk +++ b/test/bats.mk @@ -66,11 +66,11 @@ bats-check-requirements: ## Check dependencies for functional tests @$(TEST_DIR)/bin/check-requirements bats-update-tools: ## Install/update tools required for functional tests - # yq v4.40.4 - GOBIN=$(TEST_DIR)/tools go install github.com/mikefarah/yq/v4@1c3d55106075bd37df197b4bc03cb4a413fdb903 - # cfssl v1.6.4 - GOBIN=$(TEST_DIR)/tools go install github.com/cloudflare/cfssl/cmd/cfssl@b4d0d877cac528f63db39dfb62d5c96cd3a32a0b - GOBIN=$(TEST_DIR)/tools go install github.com/cloudflare/cfssl/cmd/cfssljson@b4d0d877cac528f63db39dfb62d5c96cd3a32a0b + # yq v4.43.1 + GOBIN=$(TEST_DIR)/tools go install github.com/mikefarah/yq/v4@c35ec752e38ea0c096d3c44e13cfc0797ac394d8 + # cfssl v1.6.5 + GOBIN=$(TEST_DIR)/tools go install github.com/cloudflare/cfssl/cmd/cfssl@96259aa29c9cc9b2f4e04bad7d4bc152e5405dda + GOBIN=$(TEST_DIR)/tools go install github.com/cloudflare/cfssl/cmd/cfssljson@96259aa29c9cc9b2f4e04bad7d4bc152e5405dda # Build and installs crowdsec in a local directory. Rebuilds if already exists. bats-build: bats-environment ## Build binaries for functional tests