From 84b6570554f4f5a8e7d2659a947a4375a98ce203 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Tue, 4 Jul 2023 18:46:20 +0200 Subject: [PATCH] Revert "Merge remote-tracking branch 'origin' into coraza_poc_acquis" This reverts commit 7098e971c7f25682973606291731a68d557759cc, reversing changes made to 13512891e41177323ac0a8afa4b26055f55a135a. --- .github/workflows/bats-hub.yml | 28 +- .github/workflows/bats-mysql.yml | 28 +- .github/workflows/bats-postgres.yml | 28 +- .github/workflows/bats-sqlite-coverage.yml | 28 +- .github/workflows/cache-cleanup.yaml | 35 - .github/workflows/ci-windows-build-msi.yml | 23 +- .github/workflows/docker-tests.yml | 12 + .github/workflows/go-tests-windows.yml | 23 +- .github/workflows/go-tests.yml | 31 +- .github/workflows/release_publish-package.yml | 27 +- .gitignore | 9 +- .golangci.yml | 2 +- Dockerfile | 20 +- Dockerfile.debian | 30 +- Makefile | 99 +- azure-pipelines.yml | 4 +- cmd/crowdsec-cli/alerts.go | 8 +- cmd/crowdsec-cli/capi.go | 19 +- cmd/crowdsec-cli/config_backup.go | 42 +- cmd/crowdsec-cli/config_restore.go | 24 +- cmd/crowdsec-cli/copyfile.go | 73 -- cmd/crowdsec-cli/decisions.go | 245 ++++- cmd/crowdsec-cli/decisions_import.go | 272 ----- cmd/crowdsec-cli/lapi.go | 6 +- cmd/crowdsec-cli/machines.go | 35 +- cmd/crowdsec-cli/machines_table.go | 6 +- cmd/crowdsec-cli/notifications.go | 58 +- cmd/crowdsec-cli/scenarios.go | 3 +- cmd/crowdsec-cli/support.go | 11 +- cmd/crowdsec-cli/utils.go | 6 +- cmd/crowdsec/api.go | 11 +- cmd/crowdsec/crowdsec.go | 19 +- cmd/crowdsec/main.go | 46 +- cmd/crowdsec/output.go | 15 +- cmd/crowdsec/pour.go | 4 +- cmd/crowdsec/run_in_svc_windows.go | 13 +- cmd/crowdsec/serve.go | 35 +- cmd/crowdsec/win_service.go | 4 +- debian/control | 4 +- debian/rules | 12 +- debian/templates | 2 +- docker/docker_start.sh | 2 +- docker/test/tests/test_hub_collections.py | 32 +- go.mod | 89 +- go.sum | 85 +- mk/__gmsl | 969 ------------------ mk/gmsl | 85 -- mk/gmsl.html | 733 ------------- mk/platform/unix_common.mk | 8 +- mk/platform/windows.mk | 8 +- pkg/acquisition/acquisition.go | 46 +- pkg/acquisition/acquisition_test.go | 2 +- .../modules/cloudwatch/cloudwatch.go | 16 +- pkg/acquisition/modules/docker/docker.go | 7 +- pkg/acquisition/modules/file/file.go | 22 +- pkg/acquisition/modules/file/file_test.go | 4 +- .../modules/journalctl/journalctl.go | 3 +- pkg/acquisition/modules/kafka/kafka.go | 11 +- pkg/acquisition/modules/kafka/kafka_test.go | 13 +- pkg/acquisition/modules/kinesis/kinesis.go | 37 +- .../modules/kubernetesaudit/k8s_audit.go | 14 +- pkg/acquisition/modules/s3/s3.go | 13 +- .../syslog/internal/server/syslogserver.go | 9 +- pkg/acquisition/modules/syslog/syslog.go | 5 +- pkg/apiclient/alerts_service.go | 10 +- pkg/apiclient/alerts_service_test.go | 7 +- pkg/apiclient/auth.go | 30 +- pkg/apiclient/auth_service.go | 2 + pkg/apiclient/auth_service_test.go | 5 +- pkg/apiclient/client.go | 7 +- pkg/apiclient/client_http_test.go | 4 +- pkg/apiclient/client_test.go | 3 +- pkg/apiclient/decisions_service.go | 7 +- pkg/apiclient/decisions_service_test.go | 7 +- pkg/apiclient/decisions_sync_service.go | 8 +- pkg/apiclient/signal.go | 5 +- pkg/apiserver/apic.go | 84 +- pkg/apiserver/apic_metrics.go | 145 --- pkg/apiserver/apic_metrics_test.go | 101 -- pkg/apiserver/apic_test.go | 84 ++ pkg/apiserver/apiserver.go | 28 +- pkg/apiserver/controllers/v1/controller.go | 6 +- pkg/apiserver/middlewares/v1/api_key.go | 10 +- pkg/apiserver/middlewares/v1/jwt.go | 2 +- pkg/apiserver/middlewares/v1/tls_auth.go | 7 +- pkg/csconfig/common.go | 3 +- pkg/csconfig/config_paths.go | 4 +- pkg/csconfig/profiles.go | 4 +- pkg/csplugin/broker.go | 12 +- pkg/csplugin/broker_suite_test.go | 18 +- pkg/csplugin/broker_test.go | 19 +- pkg/csplugin/broker_win_test.go | 2 +- pkg/csplugin/listfiles.go | 3 +- pkg/csplugin/listfiles_test.go | 10 +- pkg/csplugin/notifier.go | 3 +- pkg/csplugin/utils.go | 11 +- pkg/csplugin/utils_windows.go | 35 +- pkg/csplugin/watcher.go | 3 +- pkg/csplugin/watcher_test.go | 17 +- pkg/csprofiles/csprofiles.go | 9 +- pkg/csprofiles/csprofiles_test.go | 7 +- pkg/cticlient/example/fire.go | 3 - pkg/cticlient/types.go | 2 +- pkg/cwhub/cwhub.go | 13 +- pkg/cwhub/cwhub_test.go | 18 +- pkg/cwhub/download.go | 5 +- pkg/cwhub/helpers.go | 16 +- pkg/cwhub/helpers_test.go | 6 +- pkg/cwhub/loader.go | 7 +- pkg/database/alerts.go | 35 +- pkg/database/bouncers.go | 3 +- pkg/database/config.go | 3 +- pkg/database/decisions.go | 6 +- pkg/database/machines.go | 4 +- pkg/exprhelpers/jsonextract.go | 4 +- pkg/hubtest/hubtest.go | 3 +- pkg/hubtest/hubtest_item.go | 3 +- pkg/hubtest/parser_assert.go | 8 +- pkg/hubtest/scenario_assert.go | 10 +- pkg/leakybucket/README.md | 120 +-- pkg/leakybucket/bayesian.go | 163 --- pkg/leakybucket/bucket.go | 4 - pkg/leakybucket/manager_load.go | 29 - pkg/leakybucket/manager_load_test.go | 22 - pkg/leakybucket/overflows.go | 21 +- .../guillotine-bayesian-bucket/bucket.yaml | 21 - .../guillotine-bayesian-bucket/scenarios.yaml | 1 - .../guillotine-bayesian-bucket/test.json | 50 - .../multiple-bayesian-bucket/bucket.yaml | 21 - .../multiple-bayesian-bucket/scenarios.yaml | 1 - .../tests/multiple-bayesian-bucket/test.json | 64 -- .../tests/simple-bayesian-bucket/bucket.yaml | 19 - .../simple-bayesian-bucket/scenarios.yaml | 1 - .../tests/simple-bayesian-bucket/test.json | 50 - pkg/metabase/api.go | 10 +- pkg/metabase/database.go | 9 +- pkg/metabase/metabase.go | 40 +- pkg/parser/enrich.go | 5 +- pkg/parser/enrich_date.go | 3 +- pkg/parser/enrich_dns.go | 4 +- pkg/parser/enrich_geoip.go | 6 +- pkg/parser/enrich_unmarshal.go | 3 +- pkg/parser/node.go | 137 +-- pkg/parser/node_test.go | 18 +- pkg/parser/parsing_test.go | 12 +- pkg/parser/runtime.go | 32 +- pkg/parser/stage.go | 17 +- pkg/parser/unix_parser.go | 11 +- pkg/parser/whitelist.go | 1 - pkg/setup/detect.go | 7 +- pkg/setup/detect_test.go | 13 +- pkg/{cwhub => types}/dataset.go | 20 +- pkg/{cwhub => types}/dataset_test.go | 4 +- pkg/types/datasource.go | 16 - pkg/{parser => types}/grok_pattern.go | 3 +- pkg/types/ip.go | 13 +- pkg/types/utils.go | 98 +- plugins/notifications/dummy/Makefile | 5 + plugins/notifications/email/Makefile | 5 + plugins/notifications/email/go.mod | 12 +- plugins/notifications/email/go.sum | 26 +- plugins/notifications/http/Makefile | 5 + plugins/notifications/http/go.mod | 12 +- plugins/notifications/http/go.sum | 26 +- plugins/notifications/slack/Makefile | 5 + plugins/notifications/slack/go.mod | 12 +- plugins/notifications/slack/go.sum | 26 +- plugins/notifications/splunk/Makefile | 5 + plugins/notifications/splunk/go.mod | 12 +- plugins/notifications/splunk/go.sum | 26 +- rpm/SPECS/crowdsec.spec | 4 + test/README.md | 10 +- test/ansible/debug_tools.yml | 2 - test/ansible/provision_dependencies.yml | 2 - test/ansible/requirements.yml | 4 +- .../ansible/roles/make_fixture/tasks/main.yml | 4 +- test/ansible/vagrant/alma-8/Vagrantfile | 4 - test/ansible/vagrant/alma-9/Vagrantfile | 4 - test/ansible/vagrant/centos-7/Vagrantfile | 2 - test/ansible/vagrant/centos-8/Vagrantfile | 6 +- test/ansible/vagrant/centos-9/Vagrantfile | 4 - test/ansible/vagrant/common | 65 +- .../vagrant/debian-10-buster/Vagrantfile | 2 - .../vagrant/debian-11-bullseye/Vagrantfile | 2 - .../vagrant/debian-12-bookworm/Vagrantfile | 11 - .../vagrant/debian-9-stretch/Vagrantfile | 3 - .../vagrant/debian-testing/Vagrantfile | 2 - .../experimental/hardenedbsd-13/Vagrantfile | 21 +- .../experimental/hardenedbsd-13/bootstrap | 5 + .../experimental/openbsd-6/Vagrantfile | 14 - .../vagrant/experimental/openbsd-6/skip | 9 - .../experimental/openbsd-7/Vagrantfile | 24 +- .../vagrant/experimental/openbsd-7/bootstrap | 6 + test/ansible/vagrant/fedora-33/Vagrantfile | 2 - test/ansible/vagrant/fedora-34/Vagrantfile | 2 - test/ansible/vagrant/fedora-35/Vagrantfile | 2 - test/ansible/vagrant/fedora-36/Vagrantfile | 2 - test/ansible/vagrant/fedora-37/Vagrantfile | 11 - test/ansible/vagrant/fedora-38/Vagrantfile | 10 - test/ansible/vagrant/freebsd-12/Vagrantfile | 3 - test/ansible/vagrant/freebsd-13/Vagrantfile | 3 - test/ansible/vagrant/oracle-7/Vagrantfile | 3 - test/ansible/vagrant/oracle-8/Vagrantfile | 3 - test/ansible/vagrant/oracle-9/Vagrantfile | 3 - test/ansible/vagrant/rocky-8/Vagrantfile | 4 - test/ansible/vagrant/rocky-9/Vagrantfile | 4 - .../vagrant/ubuntu-16.04-xenial/Vagrantfile | 2 - .../vagrant/ubuntu-18.04-bionic/Vagrantfile | 2 - .../vagrant/ubuntu-20.04-focal/Vagrantfile | 3 - .../vagrant/ubuntu-22.04-jammy/Vagrantfile | 2 - .../vagrant/ubuntu-22.10-kinetic/Vagrantfile | 10 - .../vagrant/ubuntu-23.04-lunar/Vagrantfile | 10 - .../vagrant/wizard/centos-8/Vagrantfile | 43 +- .../ansible/vagrant/wizard/centos-8/bootstrap | 5 + test/ansible/vagrant/wizard/common | 67 -- .../wizard/debian-10-buster/Vagrantfile | 12 - .../wizard/debian-11-bullseye/Vagrantfile | 12 - .../wizard/debian-12-bookworm/Vagrantfile | 12 - .../wizard/debian-bullseye/Vagrantfile | 43 + .../vagrant/wizard/debian-bullseye/bootstrap | 5 + .../vagrant/wizard/debian-buster/Vagrantfile | 43 + .../vagrant/wizard/debian-buster/bootstrap | 5 + .../vagrant/wizard/fedora-36/Vagrantfile | 39 +- .../vagrant/wizard/fedora-36/bootstrap | 5 + .../wizard/ubuntu-22.04-jammy/Vagrantfile | 44 +- .../wizard/ubuntu-22.04-jammy/bootstrap | 5 + .../wizard/ubuntu-22.10-kinetic/Vagrantfile | 44 +- test/ansible/vars/go.yml | 2 +- test/bats-detect/openresty-deb.bats | 7 +- test/bats-detect/openresty-rpm.bats | 8 +- test/bats.mk | 3 +- test/bats/01_crowdsec.bats | 56 +- test/bats/01_cscli.bats | 8 +- test/bats/04_capi.bats | 2 +- test/bats/72_plugin_badconfig.bats | 18 +- test/bats/90_decisions.bats | 121 +-- test/bats/testdata/90_decisions/csv_decisions | 6 - test/bats/testdata/90_decisions/decisions.csv | 6 - .../bats/testdata/90_decisions/decisions.json | 42 - .../bats/testdata/90_decisions/json_decisions | 42 - test/bin/assert-crowdsec-not-running | 11 +- test/bin/check-requirements | 34 +- test/lib/setup_file.sh | 18 - test/tools/.do-not-remove | 1 - 244 files changed, 2068 insertions(+), 4722 deletions(-) delete mode 100644 .github/workflows/cache-cleanup.yaml delete mode 100644 cmd/crowdsec-cli/copyfile.go delete mode 100644 cmd/crowdsec-cli/decisions_import.go delete mode 100644 mk/__gmsl delete mode 100644 mk/gmsl delete mode 100644 mk/gmsl.html delete mode 100644 pkg/apiserver/apic_metrics.go delete mode 100644 pkg/apiserver/apic_metrics_test.go delete mode 100644 pkg/leakybucket/bayesian.go delete mode 100644 pkg/leakybucket/tests/guillotine-bayesian-bucket/bucket.yaml delete mode 100644 pkg/leakybucket/tests/guillotine-bayesian-bucket/scenarios.yaml delete mode 100644 pkg/leakybucket/tests/guillotine-bayesian-bucket/test.json delete mode 100644 pkg/leakybucket/tests/multiple-bayesian-bucket/bucket.yaml delete mode 100644 pkg/leakybucket/tests/multiple-bayesian-bucket/scenarios.yaml delete mode 100644 pkg/leakybucket/tests/multiple-bayesian-bucket/test.json delete mode 100644 pkg/leakybucket/tests/simple-bayesian-bucket/bucket.yaml delete mode 100644 pkg/leakybucket/tests/simple-bayesian-bucket/scenarios.yaml delete mode 100644 pkg/leakybucket/tests/simple-bayesian-bucket/test.json rename pkg/{cwhub => types}/dataset.go (70%) rename pkg/{cwhub => types}/dataset_test.go (94%) delete mode 100644 pkg/types/datasource.go rename pkg/{parser => types}/grok_pattern.go (99%) delete mode 100644 test/ansible/vagrant/debian-12-bookworm/Vagrantfile create mode 100755 test/ansible/vagrant/experimental/hardenedbsd-13/bootstrap delete mode 100644 test/ansible/vagrant/experimental/openbsd-6/Vagrantfile delete mode 100755 test/ansible/vagrant/experimental/openbsd-6/skip create mode 100755 test/ansible/vagrant/experimental/openbsd-7/bootstrap delete mode 100644 test/ansible/vagrant/fedora-37/Vagrantfile delete mode 100644 test/ansible/vagrant/fedora-38/Vagrantfile delete mode 100644 test/ansible/vagrant/ubuntu-22.10-kinetic/Vagrantfile delete mode 100644 test/ansible/vagrant/ubuntu-23.04-lunar/Vagrantfile create mode 100755 test/ansible/vagrant/wizard/centos-8/bootstrap delete mode 100644 test/ansible/vagrant/wizard/common delete mode 100644 test/ansible/vagrant/wizard/debian-10-buster/Vagrantfile delete mode 100644 test/ansible/vagrant/wizard/debian-11-bullseye/Vagrantfile delete mode 100644 test/ansible/vagrant/wizard/debian-12-bookworm/Vagrantfile create mode 100644 test/ansible/vagrant/wizard/debian-bullseye/Vagrantfile create mode 100755 test/ansible/vagrant/wizard/debian-bullseye/bootstrap create mode 100644 test/ansible/vagrant/wizard/debian-buster/Vagrantfile create mode 100755 test/ansible/vagrant/wizard/debian-buster/bootstrap create mode 100755 test/ansible/vagrant/wizard/fedora-36/bootstrap create mode 100755 test/ansible/vagrant/wizard/ubuntu-22.04-jammy/bootstrap delete mode 100644 test/bats/testdata/90_decisions/csv_decisions delete mode 100644 test/bats/testdata/90_decisions/decisions.csv delete mode 100644 test/bats/testdata/90_decisions/decisions.json delete mode 100644 test/bats/testdata/90_decisions/json_decisions delete mode 100644 test/tools/.do-not-remove diff --git a/.github/workflows/bats-hub.yml b/.github/workflows/bats-hub.yml index c62229cbe..026e8feaa 100644 --- a/.github/workflows/bats-hub.yml +++ b/.github/workflows/bats-hub.yml @@ -15,7 +15,7 @@ jobs: build: strategy: matrix: - go-version: ["1.20.5"] + go-version: ["1.20.4"] name: "Build + tests" runs-on: ubuntu-latest @@ -27,26 +27,40 @@ jobs: sudo chmod +w /etc/machine-id echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + - name: "Check out CrowdSec repository" uses: actions/checkout@v3 with: fetch-depth: 0 submodules: true - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 + - name: Cache Go modules + uses: actions/cache@v3 with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.go-version }}-go- - name: "Install bats dependencies" env: GOBIN: /usr/local/bin run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev + sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd + go install github.com/mikefarah/yq/v4@latest + go install github.com/cloudflare/cfssl/cmd/cfssl@master + go install github.com/cloudflare/cfssl/cmd/cfssljson@master - name: "Build crowdsec and fixture" - run: make bats-clean bats-build bats-fixture BUILD_STATIC=1 + run: make bats-clean bats-build bats-fixture - name: "Run hub tests" run: make bats-test-hub diff --git a/.github/workflows/bats-mysql.yml b/.github/workflows/bats-mysql.yml index 529c3f521..3c8825a47 100644 --- a/.github/workflows/bats-mysql.yml +++ b/.github/workflows/bats-mysql.yml @@ -14,7 +14,7 @@ jobs: build: strategy: matrix: - go-version: ["1.20.5"] + go-version: ["1.20.4"] name: "Build + tests" runs-on: ubuntu-latest @@ -34,27 +34,41 @@ jobs: sudo chmod +w /etc/machine-id echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + - name: "Check out CrowdSec repository" uses: actions/checkout@v3 with: fetch-depth: 0 submodules: true - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 + - name: Cache Go modules + uses: actions/cache@v3 with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.go-version }}-go- - name: "Install bats dependencies" env: GOBIN: /usr/local/bin run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev + sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd + go install github.com/mikefarah/yq/v4@latest + go install github.com/cloudflare/cfssl/cmd/cfssl@master + go install github.com/cloudflare/cfssl/cmd/cfssljson@master - name: "Build crowdsec and fixture" run: | - make clean bats-build bats-fixture BUILD_STATIC=1 + make clean bats-build bats-fixture env: DB_BACKEND: mysql MYSQL_HOST: 127.0.0.1 diff --git a/.github/workflows/bats-postgres.yml b/.github/workflows/bats-postgres.yml index 91e7ac103..de99a8e22 100644 --- a/.github/workflows/bats-postgres.yml +++ b/.github/workflows/bats-postgres.yml @@ -10,7 +10,7 @@ jobs: build: strategy: matrix: - go-version: ["1.20.5"] + go-version: ["1.20.4"] name: "Build + tests" runs-on: ubuntu-latest @@ -35,27 +35,41 @@ jobs: sudo chmod +w /etc/machine-id echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + - name: "Check out CrowdSec repository" uses: actions/checkout@v3 with: fetch-depth: 0 submodules: true - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 + - name: Cache Go modules + uses: actions/cache@v3 with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.go-version }}-go- - name: "Install bats dependencies" env: GOBIN: /usr/local/bin run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev + sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd + go install github.com/mikefarah/yq/v4@latest + go install github.com/cloudflare/cfssl/cmd/cfssl@master + go install github.com/cloudflare/cfssl/cmd/cfssljson@master - name: "Build crowdsec and fixture (DB_BACKEND: pgx)" run: | - make clean bats-build bats-fixture BUILD_STATIC=1 + make clean bats-build bats-fixture env: DB_BACKEND: pgx PGHOST: 127.0.0.1 diff --git a/.github/workflows/bats-sqlite-coverage.yml b/.github/workflows/bats-sqlite-coverage.yml index 17b2aac09..7b2e763b3 100644 --- a/.github/workflows/bats-sqlite-coverage.yml +++ b/.github/workflows/bats-sqlite-coverage.yml @@ -11,7 +11,7 @@ jobs: build: strategy: matrix: - go-version: ["1.20.5"] + go-version: ["1.20.4"] name: "Build + tests" runs-on: ubuntu-latest @@ -24,27 +24,41 @@ jobs: sudo chmod +w /etc/machine-id echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + - name: "Check out CrowdSec repository" uses: actions/checkout@v3 with: fetch-depth: 0 submodules: true - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 + - name: Cache Go modules + uses: actions/cache@v3 with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.go-version }}-go- - name: "Install bats dependencies" env: GOBIN: /usr/local/bin run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev + sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd + go install github.com/mikefarah/yq/v4@latest + go install github.com/cloudflare/cfssl/cmd/cfssl@master + go install github.com/cloudflare/cfssl/cmd/cfssljson@master - name: "Build crowdsec and fixture" run: | - make clean bats-build bats-fixture BUILD_STATIC=1 + make clean bats-build bats-fixture - name: "Run tests" run: make bats-test diff --git a/.github/workflows/cache-cleanup.yaml b/.github/workflows/cache-cleanup.yaml deleted file mode 100644 index d19365024..000000000 --- a/.github/workflows/cache-cleanup.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#managing-caches - -name: cleanup caches by a branch -on: - pull_request: - types: - - closed - -jobs: - cleanup: - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v3 - - - name: Cleanup - run: | - gh extension install actions/gh-actions-cache - - REPO=${{ github.repository }} - BRANCH="refs/pull/${{ github.event.pull_request.number }}/merge" - - echo "Fetching list of cache key" - cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH | cut -f 1 ) - - ## Setting this to not fail the workflow while deleting cache keys. - set +e - echo "Deleting caches..." - for cacheKey in $cacheKeysForPR - do - gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm - done - echo "Done" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-windows-build-msi.yml b/.github/workflows/ci-windows-build-msi.yml index 3b66b0c1f..b9d9722f3 100644 --- a/.github/workflows/ci-windows-build-msi.yml +++ b/.github/workflows/ci-windows-build-msi.yml @@ -23,27 +23,38 @@ jobs: build: strategy: matrix: - go-version: ["1.20.5"] + go-version: ["1.20.4"] name: Build runs-on: windows-2019 steps: + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + - name: Check out code into the Go module directory uses: actions/checkout@v3 with: fetch-depth: 0 submodules: false - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 + - name: Cache Go modules + uses: actions/cache@v3 with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.go-version }}-go- - name: Build - run: make windows_installer BUILD_RE2_WASM=1 + run: make windows_installer - name: Upload MSI uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml index 913c47662..6476d9f0a 100644 --- a/.github/workflows/docker-tests.yml +++ b/.github/workflows/docker-tests.yml @@ -30,6 +30,18 @@ jobs: with: config: .github/buildkit.toml + # - name: "Build flavor: full" + # uses: docker/build-push-action@v4 + # with: + # context: . + # file: ./Dockerfile + # tags: crowdsecurity/crowdsec:test + # target: full + # platforms: linux/amd64 + # load: true + # cache-from: type=gha + # cache-to: type=gha,mode=min + - name: "Build flavor: slim" uses: docker/build-push-action@v4 with: diff --git a/.github/workflows/go-tests-windows.yml b/.github/workflows/go-tests-windows.yml index 500fc58ce..b8e81bbe0 100644 --- a/.github/workflows/go-tests-windows.yml +++ b/.github/workflows/go-tests-windows.yml @@ -22,28 +22,39 @@ jobs: build: strategy: matrix: - go-version: ["1.20.5"] + go-version: ["1.20.4"] name: "Build + tests" runs-on: windows-2022 steps: + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + - name: Check out CrowdSec repository uses: actions/checkout@v3 with: fetch-depth: 0 submodules: false - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 + - name: Cache Go modules + uses: actions/cache@v3 with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.go-version }}-go- - name: Build run: | - make build BUILD_RE2_WASM=1 + make build - name: Run tests run: | diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 079f6c827..9b4adb4ce 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -34,7 +34,7 @@ jobs: build: strategy: matrix: - go-version: ["1.20.5"] + go-version: ["1.20.4"] name: "Build + tests" runs-on: ubuntu-latest @@ -110,30 +110,35 @@ jobs: steps: + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + - name: Check out CrowdSec repository uses: actions/checkout@v3 with: fetch-depth: 0 submodules: false - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 + - name: Cache Go modules + uses: actions/cache@v3 with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.go-version }}-go- - - name: Build and run tests, static + - name: Build and run tests run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev go install github.com/ory/go-acc@v0.2.8 go install github.com/kyoh86/richgo@v0.3.10 set -o pipefail - make build BUILD_STATIC=1 - make go-acc | richgo testfilter - - - name: Run tests again, dynamic - run: | - make clean build + make build make go-acc | richgo testfilter - name: Upload unit coverage to Codecov diff --git a/.github/workflows/release_publish-package.yml b/.github/workflows/release_publish-package.yml index 54419cc6e..600355739 100644 --- a/.github/workflows/release_publish-package.yml +++ b/.github/workflows/release_publish-package.yml @@ -14,32 +14,41 @@ jobs: build: strategy: matrix: - go-version: ["1.20.5"] + go-version: ["1.20.4"] name: Build and upload binary package runs-on: ubuntu-latest steps: + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + - name: Check out code into the Go module directory uses: actions/checkout@v3 with: fetch-depth: 0 submodules: false - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 + - name: Cache Go modules + uses: actions/cache@v3 with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.go-version }}-go- - name: Build the binaries - run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev - make vendor release BUILD_STATIC=1 + run: make release - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | tag_name="${GITHUB_REF##*/}" - hub release edit -a crowdsec-release.tgz -a vendor.tgz -m "" "$tag_name" + hub release edit -a crowdsec-release.tgz -m "" "$tag_name" diff --git a/.gitignore b/.gitignore index 8fe1778ba..4c5cb0a1f 100644 --- a/.gitignore +++ b/.gitignore @@ -15,9 +15,6 @@ *.test *.cover -# Test dependencies -test/tools/* - # VMs used for dev/test .vagrant @@ -33,10 +30,8 @@ test/coverage/* *.swp *.swo -# Dependencies are not vendored by default, but a tarball is created by "make vendor" -# and provided in the release. Used by freebsd, gentoo, etc. -vendor/ -vendor.tgz +# Dependency directories (remove the comment below to include it) +# vendor/ # crowdsec binaries cmd/crowdsec-cli/cscli diff --git a/.golangci.yml b/.golangci.yml index faa67c4bb..79900ae7d 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -65,6 +65,7 @@ linters: # - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers # - bidichk # Checks for dangerous unicode character sequences # - decorder # check declaration order and count of types, constants, variables and functions + # - depguard # Go linter that checks if package imports are in a list of acceptable packages # - dupword # checks for duplicate words in the source code # - durationcheck # check for two durations multiplied together # - errcheck # Errcheck is a program for checking for unchecked errors in go programs. These unchecked errors can be critical bugs in some cases @@ -124,7 +125,6 @@ linters: - thelper # thelper detects golang test helpers without t.Helper() call and checks the consistency of test helpers - wastedassign # wastedassign finds wasted assignment statements. - wrapcheck # Checks that errors returned from external packages are wrapped - - depguard # Go linter that checks if package imports are in a list of acceptable packages # # Recommended? (requires some work) diff --git a/Dockerfile b/Dockerfile index 2073d3c5a..da1c3ab06 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,31 +1,33 @@ # vim: set ft=dockerfile: -ARG GOVERSION=1.20.5 +ARG GOVERSION=1.20.4 FROM golang:${GOVERSION}-alpine AS build WORKDIR /go/src/crowdsec -# We like to choose the release of re2 to use, and Alpine does not ship a static version anyway. +COPY . . + +# Alpine does not ship a static version of re2, we can build it ourselves +# Later versions require 'abseil', which is likewise not available in its static form ENV RE2_VERSION=2023-03-01 # wizard.sh requires GNU coreutils -RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils pkgconfig && \ +RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils icu-static re2-dev pkgconfig && \ wget https://github.com/google/re2/archive/refs/tags/${RE2_VERSION}.tar.gz && \ tar -xzf ${RE2_VERSION}.tar.gz && \ cd re2-${RE2_VERSION} && \ + make && \ make install && \ echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \ - go install github.com/mikefarah/yq/v4@v4.34.1 - -COPY . . - -RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \ + cd - && \ + make clean release DOCKER_BUILD=1 && \ cd crowdsec-v* && \ ./wizard.sh --docker-mode && \ cd - >/dev/null && \ cscli hub update && \ cscli collections install crowdsecurity/linux && \ - cscli parsers install crowdsecurity/whitelists + cscli parsers install crowdsecurity/whitelists && \ + go install github.com/mikefarah/yq/v4@v4.31.2 # In case we need to remove agents here.. # cscli machines list -o json | yq '.[].machineId' | xargs -r cscli machines delete diff --git a/Dockerfile.debian b/Dockerfile.debian index 61778cd9e..10b06befd 100644 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -1,41 +1,32 @@ # vim: set ft=dockerfile: -ARG GOVERSION=1.20.5 +ARG GOVERSION=1.20.4 -FROM golang:${GOVERSION}-bookworm AS build +FROM golang:${GOVERSION}-bullseye AS build WORKDIR /go/src/crowdsec +COPY . . + ENV DEBIAN_FRONTEND=noninteractive ENV DEBCONF_NOWARNINGS="yes" -# We like to choose the release of re2 to use, the debian version is usually older. -ENV RE2_VERSION=2023-03-01 - # wizard.sh requires GNU coreutils RUN apt-get update && \ - apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata && \ - wget https://github.com/google/re2/archive/refs/tags/${RE2_VERSION}.tar.gz && \ - tar -xzf ${RE2_VERSION}.tar.gz && \ - cd re2-${RE2_VERSION} && \ - make && \ - make install && \ + apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata libre2-dev && \ echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \ - go install github.com/mikefarah/yq/v4@v4.34.1 - -COPY . . - -RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \ + make clean release DOCKER_BUILD=1 && \ cd crowdsec-v* && \ ./wizard.sh --docker-mode && \ cd - >/dev/null && \ cscli hub update && \ cscli collections install crowdsecurity/linux && \ - cscli parsers install crowdsecurity/whitelists + cscli parsers install crowdsecurity/whitelists && \ + go install github.com/mikefarah/yq/v4@v4.31.2 # In case we need to remove agents here.. # cscli machines list -o json | yq '.[].machineId' | xargs -r cscli machines delete -FROM debian:bookworm-slim as slim +FROM debian:bullseye-slim as slim ENV DEBIAN_FRONTEND=noninteractive ENV DEBCONF_NOWARNINGS="yes" @@ -53,6 +44,9 @@ RUN apt-get update && \ mkdir -p /staging/var/lib/crowdsec && \ mkdir -p /var/lib/crowdsec/data +RUN echo "deb http://deb.debian.org/debian bullseye-backports main" >> /etc/apt/sources.list \ + && apt-get update && apt-get install -t bullseye-backports -y libsystemd0 + COPY --from=build /go/bin/yq /usr/local/bin/yq COPY --from=build /etc/crowdsec /staging/etc/crowdsec COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec diff --git a/Makefile b/Makefile index 0fb36261f..d6f1b95f2 100644 --- a/Makefile +++ b/Makefile @@ -1,36 +1,8 @@ include mk/platform.mk -include mk/gmsl -# By default, this build requires the C++ re2 library to be installed. -# -# Debian/Ubuntu: apt install libre2-dev -# Fedora/CentOS: dnf install re2-devel -# FreeBSD: pkg install re2 -# Alpine: apk add re2-dev -# Windows: choco install re2 -# MacOS: brew install re2 - -# To build without re2, run "make BUILD_RE2_WASM=1" -# The WASM version is slower and introduces a short delay when starting a process -# (including cscli) so it is not recommended for production use. -BUILD_RE2_WASM ?= 0 - -# To build static binaries, run "make BUILD_STATIC=1". -# On some platforms, this requires additional packages -# (e.g. glibc-static and libstdc++-static on fedora, centos.. which are on the powertools/crb repository). -# If the static build fails at the link stage, it might be because the static library is not provided -# for your distribution (look for libre2.a). See the Dockerfile for an example of how to build it. -BUILD_STATIC ?= 0 - -# List of plugins to build -PLUGINS ?= $(patsubst ./plugins/notifications/%,%,$(wildcard ./plugins/notifications/*)) - -# Can be overriden, if you can deal with the consequences BUILD_REQUIRE_GO_MAJOR ?= 1 BUILD_REQUIRE_GO_MINOR ?= 20 -#-------------------------------------- - GOCMD = go GOTEST = $(GOCMD) test @@ -38,6 +10,8 @@ BUILD_CODENAME ?= alphaga CROWDSEC_FOLDER = ./cmd/crowdsec CSCLI_FOLDER = ./cmd/crowdsec-cli/ + +PLUGINS ?= $(patsubst ./plugins/notifications/%,%,$(wildcard ./plugins/notifications/*)) PLUGINS_DIR = ./plugins/notifications CROWDSEC_BIN = crowdsec$(EXT) @@ -48,14 +22,8 @@ RELDIR = crowdsec-$(BUILD_VERSION) GO_MODULE_NAME = github.com/crowdsecurity/crowdsec -# Check if a given value is considered truthy and returns "0" or "1". -# A truthy value is one of the following: "1", "yes", or "true", case-insensitive. -# -# Usage: -# ifeq ($(call bool,$(FOO)),1) -# $(info Let's foo) -# endif -bool = $(if $(filter $(call lc, $1),1 yes true),1,0) +# see if we have libre2-dev installed for C++ optimizations +RE2_CHECK := $(shell pkg-config --libs re2 2>/dev/null) #-------------------------------------- # @@ -78,30 +46,13 @@ endif GO_TAGS := netgo,osusergo,sqlite_omit_load_extension -ifeq ($(call bool,$(BUILD_RE2_WASM)),0) -ifeq ($(PKG_CONFIG),) - $(error "pkg-config is not available. Please install pkg-config.") -endif - -ifeq ($(RE2_CHECK),) -# we could detect the platform and suggest the command to install -RE2_FAIL := "libre2-dev is not installed, please install it or set BUILD_RE2_WASM=1 to use the WebAssembly version" -else +ifneq (,$(RE2_CHECK)) # += adds a space that we don't want GO_TAGS := $(GO_TAGS),re2_cgo LD_OPTS_VARS += -X '$(GO_MODULE_NAME)/pkg/cwversion.Libre2=C++' endif -endif -ifeq ($(call bool,$(BUILD_STATIC)),1) -BUILD_TYPE = static -EXTLDFLAGS := -extldflags '-static' -else -BUILD_TYPE = dynamic -EXTLDFLAGS := -endif - -export LD_OPTS=-ldflags "-s -w $(EXTLDFLAGS) $(LD_OPTS_VARS)" \ +export LD_OPTS=-ldflags "-s -w -extldflags '-static' $(LD_OPTS_VARS)" \ -trimpath -tags $(GO_TAGS) ifneq (,$(TEST_COVERAGE)) @@ -113,15 +64,12 @@ endif .PHONY: build build: pre-build goversion crowdsec cscli plugins -# Sanity checks and build information .PHONY: pre-build pre-build: - $(info Building $(BUILD_VERSION) ($(BUILD_TAG)) $(BUILD_TYPE) for $(GOOS)/$(GOARCH)) - -ifneq (,$(RE2_FAIL)) - $(error $(RE2_FAIL)) +ifdef BUILD_STATIC + $(warning WARNING: The BUILD_STATIC variable is deprecated and has no effect. Builds are static by default since v1.5.0.) endif - + $(info Building $(BUILD_VERSION) ($(BUILD_TAG)) for $(GOOS)/$(GOARCH)) ifneq (,$(RE2_CHECK)) $(info Using C++ regexp library) else @@ -165,7 +113,6 @@ testclean: bats-clean @$(RM) pkg/cwhub/install $(WIN_IGNORE_ERR) @$(RM) pkg/types/example.txt $(WIN_IGNORE_ERR) -# for the tests with localstack export AWS_ENDPOINT_FORCE=http://localhost:4566 export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY @@ -173,18 +120,15 @@ export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY testenv: @echo 'NOTE: You need Docker, docker-compose and run "make localstack" in a separate shell ("make localstack-stop" to terminate it)' -# run the tests with localstack .PHONY: test test: testenv goversion $(GOTEST) $(LD_OPTS) ./... -# run the tests with localstack and coverage .PHONY: go-acc go-acc: testenv goversion go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models -- $(LD_OPTS) | \ sed 's/ *coverage:.*of statements in.*//' -# mock AWS services .PHONY: localstack localstack: docker-compose -f test/localstack/docker-compose.yml up @@ -193,27 +137,13 @@ localstack: localstack-stop: docker-compose -f test/localstack/docker-compose.yml down -# list of plugins that contain go.mod -PLUGIN_VENDOR = $(foreach plugin,$(PLUGINS),$(shell if [ -f $(PLUGINS_DIR)/$(plugin)/go.mod ]; then echo $(PLUGINS_DIR)/$(plugin); fi)) - -# build vendor.tgz to be distributed with the release .PHONY: vendor vendor: - $(foreach plugin_dir,$(PLUGIN_VENDOR), \ - cd $(plugin_dir) >/dev/null && \ - $(GOCMD) mod vendor && \ - cd - >/dev/null; \ + @echo "Vendoring dependencies" + @$(GOCMD) mod vendor + @$(foreach plugin,$(PLUGINS), \ + $(MAKE) -C $(PLUGINS_DIR)/$(plugin) vendor $(MAKE_FLAGS); \ ) - $(GOCMD) mod vendor - tar -czf vendor.tgz vendor $(foreach plugin_dir,$(PLUGIN_VENDOR),$(plugin_dir)/vendor) - -# remove vendor directories and vendor.tgz -.PHONY: vendor-remove -vendor-remove: - $(foreach plugin_dir,$(PLUGIN_VENDOR), \ - $(RM) $(plugin_dir)/vendor; \ - ) - $(RM) vendor vendor.tgz .PHONY: package package: @@ -244,16 +174,13 @@ else @if (Test-Path -Path $(RELDIR)) { echo "$(RELDIR) already exists, abort" ; exit 1 ; } endif -# build a release tarball .PHONY: release release: check_release build package -# build the windows installer .PHONY: windows_installer windows_installer: build @.\make_installer.ps1 -version $(BUILD_VERSION) -# build the chocolatey package .PHONY: chocolatey chocolatey: windows_installer @.\make_chocolatey.ps1 -version $(BUILD_VERSION) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b1564b375..c529ee2fa 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -27,7 +27,7 @@ stages: - task: GoTool@0 displayName: "Install Go 1.20" inputs: - version: '1.20.5' + version: '1.20.4' - pwsh: | choco install -y make @@ -38,7 +38,7 @@ stages: pwsh: true #we are not calling make windows_installer because we want to sign the binaries before they are added to the MSI script: | - make build BUILD_RE2_WASM=1 + make build - task: AzureKeyVault@2 inputs: azureSubscription: 'Azure subscription 1(8a93ab40-7e99-445e-ad47-0f6a3e2ef546)' diff --git a/cmd/crowdsec-cli/alerts.go b/cmd/crowdsec-cli/alerts.go index 6abe3db5a..25cb26515 100644 --- a/cmd/crowdsec-cli/alerts.go +++ b/cmd/crowdsec-cli/alerts.go @@ -15,6 +15,7 @@ import ( "github.com/fatih/color" "github.com/go-openapi/strfmt" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "gopkg.in/yaml.v2" @@ -154,6 +155,7 @@ var alertTemplate = ` ` + func DisplayOneAlert(alert *models.Alert, withDetail bool) error { if csConfig.Cscli.Output == "human" { tmpl, err := template.New("alert").Parse(alertTemplate) @@ -209,11 +211,11 @@ func NewAlertsCmd() *cobra.Command { PersistentPreRunE: func(cmd *cobra.Command, args []string) error { var err error if err := csConfig.LoadAPIClient(); err != nil { - return fmt.Errorf("loading api client: %w", err) + return errors.Wrap(err, "loading api client") } apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL) if err != nil { - return fmt.Errorf("parsing api url %s: %w", apiURL, err) + return errors.Wrapf(err, "parsing api url %s", apiURL) } Client, err = apiclient.NewClient(&apiclient.Config{ MachineID: csConfig.API.Client.Credentials.Login, @@ -224,7 +226,7 @@ func NewAlertsCmd() *cobra.Command { }) if err != nil { - return fmt.Errorf("new api client: %w", err) + return errors.Wrap(err, "new api client") } return nil }, diff --git a/cmd/crowdsec-cli/capi.go b/cmd/crowdsec-cli/capi.go index af6e9c2e8..e67d33ce4 100644 --- a/cmd/crowdsec-cli/capi.go +++ b/cmd/crowdsec-cli/capi.go @@ -6,11 +6,6 @@ import ( "net/url" "os" - "github.com/go-openapi/strfmt" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "gopkg.in/yaml.v2" - "github.com/crowdsecurity/go-cs-lib/pkg/version" "github.com/crowdsecurity/crowdsec/pkg/apiclient" @@ -19,6 +14,11 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/fflag" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/go-openapi/strfmt" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "gopkg.in/yaml.v2" ) const CAPIBaseURL string = "https://api.crowdsec.net/" @@ -31,11 +31,8 @@ func NewCapiCmd() *cobra.Command { Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := csConfig.LoadAPIServer(); err != nil { - return fmt.Errorf("local API is disabled, please run this command on the local API machine: %w", err) - } - if csConfig.DisableAPI { - return nil + if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI { + return errors.Wrap(err, "Local API is disabled, please run this command on the local API machine") } if csConfig.API.Server.OnlineClient == nil { log.Fatalf("no configuration for Central API in '%s'", *csConfig.FilePath) @@ -136,7 +133,7 @@ func NewCapiStatusCmd() *cobra.Command { Run: func(cmd *cobra.Command, args []string) { var err error if csConfig.API.Server == nil { - log.Fatal("There is no configuration on 'api.server:'") + log.Fatalln("There is no configuration on 'api.server:'") } if csConfig.API.Server.OnlineClient == nil { log.Fatalf("Please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath) diff --git a/cmd/crowdsec-cli/config_backup.go b/cmd/crowdsec-cli/config_backup.go index 717fc990b..30cf729fe 100644 --- a/cmd/crowdsec-cli/config_backup.go +++ b/cmd/crowdsec-cli/config_backup.go @@ -5,9 +5,11 @@ import ( "os" "path/filepath" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) @@ -32,17 +34,17 @@ func backupConfigToDirectory(dirPath string) error { /*if parent directory doesn't exist, bail out. create final dir with Mkdir*/ parentDir := filepath.Dir(dirPath) if _, err := os.Stat(parentDir); err != nil { - return fmt.Errorf("while checking parent directory %s existence: %w", parentDir, err) + return errors.Wrapf(err, "while checking parent directory %s existence", parentDir) } if err = os.Mkdir(dirPath, 0o700); err != nil { - return fmt.Errorf("while creating %s: %w", dirPath, err) + return errors.Wrapf(err, "while creating %s", dirPath) } if csConfig.ConfigPaths.SimulationFilePath != "" { backupSimulation := filepath.Join(dirPath, "simulation.yaml") - if err = CopyFile(csConfig.ConfigPaths.SimulationFilePath, backupSimulation); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", csConfig.ConfigPaths.SimulationFilePath, backupSimulation, err) + if err = types.CopyFile(csConfig.ConfigPaths.SimulationFilePath, backupSimulation); err != nil { + return errors.Wrapf(err, "failed copy %s to %s", csConfig.ConfigPaths.SimulationFilePath, backupSimulation) } log.Infof("Saved simulation to %s", backupSimulation) @@ -54,14 +56,14 @@ func backupConfigToDirectory(dirPath string) error { */ if csConfig.Crowdsec != nil && csConfig.Crowdsec.AcquisitionFilePath != "" { backupAcquisition := filepath.Join(dirPath, "acquis.yaml") - if err = CopyFile(csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil { - return fmt.Errorf("failed copy %s to %s: %s", csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition, err) + if err = types.CopyFile(csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil { + return fmt.Errorf("failed copy %s to %s : %s", csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition, err) } } acquisBackupDir := filepath.Join(dirPath, "acquis") if err = os.Mkdir(acquisBackupDir, 0o700); err != nil { - return fmt.Errorf("error while creating %s: %s", acquisBackupDir, err) + return fmt.Errorf("error while creating %s : %s", acquisBackupDir, err) } if csConfig.Crowdsec != nil && len(csConfig.Crowdsec.AcquisitionFiles) > 0 { @@ -73,11 +75,11 @@ func backupConfigToDirectory(dirPath string) error { targetFname, err := filepath.Abs(filepath.Join(acquisBackupDir, filepath.Base(acquisFile))) if err != nil { - return fmt.Errorf("while saving %s to %s: %w", acquisFile, acquisBackupDir, err) + return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir) } - if err = CopyFile(acquisFile, targetFname); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", acquisFile, targetFname, err) + if err = types.CopyFile(acquisFile, targetFname); err != nil { + return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err) } log.Infof("Saved acquis %s to %s", acquisFile, targetFname) @@ -86,8 +88,8 @@ func backupConfigToDirectory(dirPath string) error { if ConfigFilePath != "" { backupMain := fmt.Sprintf("%s/config.yaml", dirPath) - if err = CopyFile(ConfigFilePath, backupMain); err != nil { - return fmt.Errorf("failed copy %s to %s: %s", ConfigFilePath, backupMain, err) + if err = types.CopyFile(ConfigFilePath, backupMain); err != nil { + return fmt.Errorf("failed copy %s to %s : %s", ConfigFilePath, backupMain, err) } log.Infof("Saved default yaml to %s", backupMain) @@ -95,8 +97,8 @@ func backupConfigToDirectory(dirPath string) error { if csConfig.API != nil && csConfig.API.Server != nil && csConfig.API.Server.OnlineClient != nil && csConfig.API.Server.OnlineClient.CredentialsFilePath != "" { backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath) - if err = CopyFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds); err != nil { - return fmt.Errorf("failed copy %s to %s: %s", csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds, err) + if err = types.CopyFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds); err != nil { + return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds, err) } log.Infof("Saved online API credentials to %s", backupCAPICreds) @@ -104,8 +106,8 @@ func backupConfigToDirectory(dirPath string) error { if csConfig.API != nil && csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" { backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath) - if err = CopyFile(csConfig.API.Client.CredentialsFilePath, backupLAPICreds); err != nil { - return fmt.Errorf("failed copy %s to %s: %s", csConfig.API.Client.CredentialsFilePath, backupLAPICreds, err) + if err = types.CopyFile(csConfig.API.Client.CredentialsFilePath, backupLAPICreds); err != nil { + return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Client.CredentialsFilePath, backupLAPICreds, err) } log.Infof("Saved local API credentials to %s", backupLAPICreds) @@ -113,20 +115,21 @@ func backupConfigToDirectory(dirPath string) error { if csConfig.API != nil && csConfig.API.Server != nil && csConfig.API.Server.ProfilesPath != "" { backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath) - if err = CopyFile(csConfig.API.Server.ProfilesPath, backupProfiles); err != nil { - return fmt.Errorf("failed copy %s to %s: %s", csConfig.API.Server.ProfilesPath, backupProfiles, err) + if err = types.CopyFile(csConfig.API.Server.ProfilesPath, backupProfiles); err != nil { + return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Server.ProfilesPath, backupProfiles, err) } log.Infof("Saved profiles to %s", backupProfiles) } if err = BackupHub(dirPath); err != nil { - return fmt.Errorf("failed to backup hub config: %s", err) + return fmt.Errorf("failed to backup hub config : %s", err) } return nil } + func runConfigBackup(cmd *cobra.Command, args []string) error { if err := csConfig.LoadHub(); err != nil { return err @@ -144,6 +147,7 @@ func runConfigBackup(cmd *cobra.Command, args []string) error { return nil } + func NewConfigBackupCmd() *cobra.Command { cmdConfigBackup := &cobra.Command{ Use: `backup "directory"`, diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index 55ab7aa9b..79d36d428 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -7,11 +7,13 @@ import ( "os" "path/filepath" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "gopkg.in/yaml.v2" "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) @@ -36,7 +38,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { backupMain := fmt.Sprintf("%s/config.yaml", dirPath) if _, err = os.Stat(backupMain); err == nil { if csConfig.ConfigPaths != nil && csConfig.ConfigPaths.ConfigDir != "" { - if err = CopyFile(backupMain, fmt.Sprintf("%s/config.yaml", csConfig.ConfigPaths.ConfigDir)); err != nil { + if err = types.CopyFile(backupMain, fmt.Sprintf("%s/config.yaml", csConfig.ConfigPaths.ConfigDir)); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupMain, csConfig.ConfigPaths.ConfigDir, err) } } @@ -49,21 +51,21 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath) if _, err = os.Stat(backupCAPICreds); err == nil { - if err = CopyFile(backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath); err != nil { + if err = types.CopyFile(backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath, err) } } backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath) if _, err = os.Stat(backupLAPICreds); err == nil { - if err = CopyFile(backupLAPICreds, csConfig.API.Client.CredentialsFilePath); err != nil { + if err = types.CopyFile(backupLAPICreds, csConfig.API.Client.CredentialsFilePath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupLAPICreds, csConfig.API.Client.CredentialsFilePath, err) } } backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath) if _, err = os.Stat(backupProfiles); err == nil { - if err = CopyFile(backupProfiles, csConfig.API.Server.ProfilesPath); err != nil { + if err = types.CopyFile(backupProfiles, csConfig.API.Server.ProfilesPath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupProfiles, csConfig.API.Server.ProfilesPath, err) } } @@ -104,7 +106,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { backupSimulation := fmt.Sprintf("%s/simulation.yaml", dirPath) if _, err = os.Stat(backupSimulation); err == nil { - if err = CopyFile(backupSimulation, csConfig.ConfigPaths.SimulationFilePath); err != nil { + if err = types.CopyFile(backupSimulation, csConfig.ConfigPaths.SimulationFilePath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupSimulation, csConfig.ConfigPaths.SimulationFilePath, err) } } @@ -121,7 +123,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { if _, err = os.Stat(backupAcquisition); err == nil { log.Debugf("restoring backup'ed %s", backupAcquisition) - if err = CopyFile(backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath); err != nil { + if err = types.CopyFile(backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath, err) } } @@ -132,12 +134,12 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { for _, acquisFile := range acquisFiles { targetFname, err := filepath.Abs(csConfig.Crowdsec.AcquisitionDirPath + "/" + filepath.Base(acquisFile)) if err != nil { - return fmt.Errorf("while saving %s to %s: %w", acquisFile, targetFname, err) + return errors.Wrapf(err, "while saving %s to %s", acquisFile, targetFname) } log.Debugf("restoring %s to %s", acquisFile, targetFname) - if err = CopyFile(acquisFile, targetFname); err != nil { + if err = types.CopyFile(acquisFile, targetFname); err != nil { return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err) } } @@ -155,10 +157,10 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { targetFname, err := filepath.Abs(filepath.Join(acquisBackupDir, filepath.Base(acquisFile))) if err != nil { - return fmt.Errorf("while saving %s to %s: %w", acquisFile, acquisBackupDir, err) + return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir) } - if err = CopyFile(acquisFile, targetFname); err != nil { + if err = types.CopyFile(acquisFile, targetFname); err != nil { return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err) } @@ -173,6 +175,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { return nil } + func runConfigRestore(cmd *cobra.Command, args []string) error { flags := cmd.Flags() @@ -197,6 +200,7 @@ func runConfigRestore(cmd *cobra.Command, args []string) error { return nil } + func NewConfigRestoreCmd() *cobra.Command { cmdConfigRestore := &cobra.Command{ Use: `restore "directory"`, diff --git a/cmd/crowdsec-cli/copyfile.go b/cmd/crowdsec-cli/copyfile.go deleted file mode 100644 index 4de6cd6e2..000000000 --- a/cmd/crowdsec-cli/copyfile.go +++ /dev/null @@ -1,73 +0,0 @@ -package main - -import ( - "fmt" - "io" - "os" - "path/filepath" - - log "github.com/sirupsen/logrus" -) - - -/*help to copy the file, ioutil doesn't offer the feature*/ - -func copyFileContents(src, dst string) (err error) { - in, err := os.Open(src) - if err != nil { - return - } - defer in.Close() - out, err := os.Create(dst) - if err != nil { - return - } - defer func() { - cerr := out.Close() - if err == nil { - err = cerr - } - }() - if _, err = io.Copy(out, in); err != nil { - return - } - err = out.Sync() - return -} - -/*copy the file, ioutile doesn't offer the feature*/ -func CopyFile(sourceSymLink, destinationFile string) (err error) { - sourceFile, err := filepath.EvalSymlinks(sourceSymLink) - if err != nil { - log.Infof("Not a symlink : %s", err) - sourceFile = sourceSymLink - } - - sourceFileStat, err := os.Stat(sourceFile) - if err != nil { - return - } - if !sourceFileStat.Mode().IsRegular() { - // cannot copy non-regular files (e.g., directories, - // symlinks, devices, etc.) - return fmt.Errorf("copyFile: non-regular source file %s (%q)", sourceFileStat.Name(), sourceFileStat.Mode().String()) - } - destinationFileStat, err := os.Stat(destinationFile) - if err != nil { - if !os.IsNotExist(err) { - return - } - } else { - if !(destinationFileStat.Mode().IsRegular()) { - return fmt.Errorf("copyFile: non-regular destination file %s (%q)", destinationFileStat.Name(), destinationFileStat.Mode().String()) - } - if os.SameFile(sourceFileStat, destinationFileStat) { - return - } - } - if err = os.Link(sourceFile, destinationFile); err != nil { - err = copyFileContents(sourceFile, destinationFile) - } - return -} - diff --git a/cmd/crowdsec-cli/decisions.go b/cmd/crowdsec-cli/decisions.go index ce3d0e46e..f2f3efcf8 100644 --- a/cmd/crowdsec-cli/decisions.go +++ b/cmd/crowdsec-cli/decisions.go @@ -7,15 +7,19 @@ import ( "fmt" "net/url" "os" + "path/filepath" "strconv" "strings" "time" "github.com/fatih/color" "github.com/go-openapi/strfmt" + "github.com/jszwec/csvutil" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/crowdsecurity/go-cs-lib/pkg/ptr" "github.com/crowdsecurity/go-cs-lib/pkg/version" "github.com/crowdsecurity/crowdsec/pkg/apiclient" @@ -108,12 +112,12 @@ func NewDecisionsCmd() *cobra.Command { DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { if err := csConfig.LoadAPIClient(); err != nil { - return fmt.Errorf("loading api client: %w", err) + return errors.Wrap(err, "loading api client") } password := strfmt.Password(csConfig.API.Client.Credentials.Password) apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL) if err != nil { - return fmt.Errorf("parsing api url %s: %w", csConfig.API.Client.Credentials.URL, err) + return errors.Wrapf(err, "parsing api url %s", csConfig.API.Client.Credentials.URL) } Client, err = apiclient.NewClient(&apiclient.Config{ MachineID: csConfig.API.Client.Credentials.Login, @@ -123,7 +127,7 @@ func NewDecisionsCmd() *cobra.Command { VersionPrefix: "v1", }) if err != nil { - return fmt.Errorf("creating api client: %w", err) + return errors.Wrap(err, "creating api client") } return nil }, @@ -165,11 +169,11 @@ cscli decisions list -t ban `, Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { + Run: func(cmd *cobra.Command, args []string) { var err error /*take care of shorthand options*/ - if err = manageCliDecisionAlerts(filter.IPEquals, filter.RangeEquals, filter.ScopeEquals, filter.ValueEquals); err != nil { - return err + if err := manageCliDecisionAlerts(filter.IPEquals, filter.RangeEquals, filter.ScopeEquals, filter.ValueEquals); err != nil { + log.Fatalf("%s", err) } filter.ActiveDecisionEquals = new(bool) *filter.ActiveDecisionEquals = true @@ -185,7 +189,7 @@ cscli decisions list -t ban days, err := strconv.Atoi(realDuration) if err != nil { printHelp(cmd) - return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until) + log.Fatalf("Can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until) } *filter.Until = fmt.Sprintf("%d%s", days*24, "h") } @@ -198,7 +202,7 @@ cscli decisions list -t ban days, err := strconv.Atoi(realDuration) if err != nil { printHelp(cmd) - return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Since) + log.Fatalf("Can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until) } *filter.Since = fmt.Sprintf("%d%s", days*24, "h") } @@ -234,15 +238,13 @@ cscli decisions list -t ban alerts, _, err := Client.Alerts.List(context.Background(), filter) if err != nil { - return fmt.Errorf("unable to retrieve decisions: %w", err) + log.Fatalf("Unable to list decisions : %v", err) } err = DecisionsToTable(alerts, printMachine) if err != nil { - return fmt.Errorf("unable to print decisions: %w", err) + log.Fatalf("unable to list decisions : %v", err) } - - return nil }, } cmdDecisionsList.Flags().SortFlags = false @@ -286,7 +288,7 @@ cscli decisions add --scope username --value foobar /*TBD : fix long and example*/ Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { + Run: func(cmd *cobra.Command, args []string) { var err error alerts := models.AddAlertsRequest{} origin := types.CscliOrigin @@ -301,7 +303,7 @@ cscli decisions add --scope username --value foobar /*take care of shorthand options*/ if err := manageCliDecisionAlerts(&addIP, &addRange, &addScope, &addValue); err != nil { - return err + log.Fatalf("%s", err) } if addIP != "" { @@ -312,7 +314,7 @@ cscli decisions add --scope username --value foobar addScope = types.Range } else if addValue == "" { printHelp(cmd) - return fmt.Errorf("Missing arguments, a value is required (--ip, --range or --scope and --value)") + log.Fatalf("Missing arguments, a value is required (--ip, --range or --scope and --value)") } if addReason == "" { @@ -355,11 +357,10 @@ cscli decisions add --scope username --value foobar _, _, err = Client.Alerts.Add(context.Background(), alerts) if err != nil { - return err + log.Fatal(err) } log.Info("Decision successfully added") - return nil }, } @@ -400,27 +401,25 @@ cscli decisions delete --id 42 cscli decisions delete --type captcha `, /*TBD : refaire le Long/Example*/ - PreRunE: func(cmd *cobra.Command, args []string) error { + PreRun: func(cmd *cobra.Command, args []string) { if delDecisionAll { - return nil + return } if *delFilter.ScopeEquals == "" && *delFilter.ValueEquals == "" && *delFilter.TypeEquals == "" && *delFilter.IPEquals == "" && *delFilter.RangeEquals == "" && *delFilter.ScenarioEquals == "" && *delFilter.OriginEquals == "" && delDecisionId == "" { cmd.Usage() - return fmt.Errorf("at least one filter or --all must be specified") + log.Fatalln("At least one filter or --all must be specified") } - - return nil }, - RunE: func(cmd *cobra.Command, args []string) error { + Run: func(cmd *cobra.Command, args []string) { var err error var decisions *models.DeleteDecisionResponse /*take care of shorthand options*/ - if err = manageCliDecisionAlerts(delFilter.IPEquals, delFilter.RangeEquals, delFilter.ScopeEquals, delFilter.ValueEquals); err != nil { - return err + if err := manageCliDecisionAlerts(delFilter.IPEquals, delFilter.RangeEquals, delFilter.ScopeEquals, delFilter.ValueEquals); err != nil { + log.Fatalf("%s", err) } if *delFilter.ScopeEquals == "" { delFilter.ScopeEquals = nil @@ -450,19 +449,18 @@ cscli decisions delete --type captcha if delDecisionId == "" { decisions, _, err = Client.Decisions.Delete(context.Background(), delFilter) if err != nil { - return fmt.Errorf("Unable to delete decisions: %v", err) + log.Fatalf("Unable to delete decisions : %v", err) } } else { if _, err = strconv.Atoi(delDecisionId); err != nil { - return fmt.Errorf("id '%s' is not an integer: %v", delDecisionId, err) + log.Fatalf("id '%s' is not an integer: %v", delDecisionId, err) } decisions, _, err = Client.Decisions.DeleteOne(context.Background(), delDecisionId) if err != nil { - return fmt.Errorf("Unable to delete decision: %v", err) + log.Fatalf("Unable to delete decision : %v", err) } } log.Infof("%s decision(s) deleted", decisions.NbDeleted) - return nil }, } @@ -480,3 +478,192 @@ cscli decisions delete --type captcha return cmdDecisionsDelete } + +func NewDecisionsImportCmd() *cobra.Command { + var ( + defaultDuration = "4h" + defaultScope = "ip" + defaultType = "ban" + defaultReason = "manual" + importDuration string + importScope string + importReason string + importType string + importFile string + batchSize int + ) + + var cmdDecisionImport = &cobra.Command{ + Use: "import [options]", + Short: "Import decisions from json or csv file", + Long: "expected format :\n" + + "csv : any of duration,origin,reason,scope,type,value, with a header line\n" + + `json : {"duration" : "24h", "origin" : "my-list", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"}`, + DisableAutoGenTag: true, + Example: `decisions.csv : +duration,scope,value +24h,ip,1.2.3.4 + +cscsli decisions import -i decisions.csv + +decisions.json : +[{"duration" : "4h", "scope" : "ip", "type" : "ban", "value" : "1.2.3.4"}] +`, + Run: func(cmd *cobra.Command, args []string) { + if importFile == "" { + log.Fatalf("Please provide a input file containing decisions with -i flag") + } + csvData, err := os.ReadFile(importFile) + if err != nil { + log.Fatalf("unable to open '%s': %s", importFile, err) + } + type decisionRaw struct { + Duration string `csv:"duration,omitempty" json:"duration,omitempty"` + Origin string `csv:"origin,omitempty" json:"origin,omitempty"` + Scenario string `csv:"reason,omitempty" json:"reason,omitempty"` + Scope string `csv:"scope,omitempty" json:"scope,omitempty"` + Type string `csv:"type,omitempty" json:"type,omitempty"` + Value string `csv:"value" json:"value"` + } + var decisionsListRaw []decisionRaw + switch fileFormat := filepath.Ext(importFile); fileFormat { + case ".json": + if err := json.Unmarshal(csvData, &decisionsListRaw); err != nil { + log.Fatalf("unable to unmarshall json: '%s'", err) + } + case ".csv": + if err := csvutil.Unmarshal(csvData, &decisionsListRaw); err != nil { + log.Fatalf("unable to unmarshall csv: '%s'", err) + } + default: + log.Fatalf("file format not supported for '%s'. supported format are 'json' and 'csv'", importFile) + } + + decisionsList := make([]*models.Decision, 0) + for i, decisionLine := range decisionsListRaw { + line := i + 2 + if decisionLine.Value == "" { + log.Fatalf("please provide a 'value' in your csv line %d", line) + } + /*deal with defaults and cli-override*/ + if decisionLine.Duration == "" { + decisionLine.Duration = defaultDuration + log.Debugf("No 'duration' line %d, using default value: '%s'", line, defaultDuration) + } + if importDuration != "" { + decisionLine.Duration = importDuration + log.Debugf("'duration' line %d, using supplied value: '%s'", line, importDuration) + } + decisionLine.Origin = types.CscliImportOrigin + + if decisionLine.Scenario == "" { + decisionLine.Scenario = defaultReason + log.Debugf("No 'reason' line %d, using value: '%s'", line, decisionLine.Scenario) + } + if importReason != "" { + decisionLine.Scenario = importReason + log.Debugf("No 'reason' line %d, using supplied value: '%s'", line, importReason) + } + if decisionLine.Type == "" { + decisionLine.Type = defaultType + log.Debugf("No 'type' line %d, using default value: '%s'", line, decisionLine.Type) + } + if importType != "" { + decisionLine.Type = importType + log.Debugf("'type' line %d, using supplied value: '%s'", line, importType) + } + if decisionLine.Scope == "" { + decisionLine.Scope = defaultScope + log.Debugf("No 'scope' line %d, using default value: '%s'", line, decisionLine.Scope) + } + if importScope != "" { + decisionLine.Scope = importScope + log.Debugf("'scope' line %d, using supplied value: '%s'", line, importScope) + } + decision := models.Decision{ + Value: ptr.Of(decisionLine.Value), + Duration: ptr.Of(decisionLine.Duration), + Origin: ptr.Of(decisionLine.Origin), + Scenario: ptr.Of(decisionLine.Scenario), + Type: ptr.Of(decisionLine.Type), + Scope: ptr.Of(decisionLine.Scope), + Simulated: new(bool), + } + decisionsList = append(decisionsList, &decision) + } + alerts := models.AddAlertsRequest{} + + if batchSize > 0 { + for i := 0; i < len(decisionsList); i += batchSize { + end := i + batchSize + if end > len(decisionsList) { + end = len(decisionsList) + } + decisionBatch := decisionsList[i:end] + importAlert := models.Alert{ + CreatedAt: time.Now().UTC().Format(time.RFC3339), + Scenario: ptr.Of(fmt.Sprintf("import %s : %d IPs", importFile, len(decisionBatch))), + + Message: ptr.Of(""), + Events: []*models.Event{}, + Source: &models.Source{ + Scope: ptr.Of(""), + Value: ptr.Of(""), + }, + StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), + StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), + Capacity: ptr.Of(int32(0)), + Simulated: ptr.Of(false), + EventsCount: ptr.Of(int32(len(decisionBatch))), + Leakspeed: ptr.Of(""), + ScenarioHash: ptr.Of(""), + ScenarioVersion: ptr.Of(""), + Decisions: decisionBatch, + } + alerts = append(alerts, &importAlert) + } + } else { + importAlert := models.Alert{ + CreatedAt: time.Now().UTC().Format(time.RFC3339), + Scenario: ptr.Of(fmt.Sprintf("import %s : %d IPs", importFile, len(decisionsList))), + Message: ptr.Of(""), + Events: []*models.Event{}, + Source: &models.Source{ + Scope: ptr.Of(""), + Value: ptr.Of(""), + }, + StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), + StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), + Capacity: ptr.Of(int32(0)), + Simulated: ptr.Of(false), + EventsCount: ptr.Of(int32(len(decisionsList))), + Leakspeed: ptr.Of(""), + ScenarioHash: ptr.Of(""), + ScenarioVersion: ptr.Of(""), + Decisions: decisionsList, + } + alerts = append(alerts, &importAlert) + } + + if len(decisionsList) > 1000 { + log.Infof("You are about to add %d decisions, this may take a while", len(decisionsList)) + } + + _, _, err = Client.Alerts.Add(context.Background(), alerts) + if err != nil { + log.Fatal(err) + } + log.Infof("%d decisions successfully imported", len(decisionsList)) + }, + } + + cmdDecisionImport.Flags().SortFlags = false + cmdDecisionImport.Flags().StringVarP(&importFile, "input", "i", "", "Input file") + cmdDecisionImport.Flags().StringVarP(&importDuration, "duration", "d", "", "Decision duration (ie. 1h,4h,30m)") + cmdDecisionImport.Flags().StringVar(&importScope, "scope", types.Ip, "Decision scope (ie. ip,range,username)") + cmdDecisionImport.Flags().StringVarP(&importReason, "reason", "R", "", "Decision reason (ie. scenario-name)") + cmdDecisionImport.Flags().StringVarP(&importType, "type", "t", "", "Decision type (ie. ban,captcha,throttle)") + cmdDecisionImport.Flags().IntVar(&batchSize, "batch", 0, "Split import in batches of N decisions") + + return cmdDecisionImport +} diff --git a/cmd/crowdsec-cli/decisions_import.go b/cmd/crowdsec-cli/decisions_import.go deleted file mode 100644 index 6a47a96b3..000000000 --- a/cmd/crowdsec-cli/decisions_import.go +++ /dev/null @@ -1,272 +0,0 @@ -package main - -import ( - "bufio" - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "os" - "strings" - "time" - - "github.com/jszwec/csvutil" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - - "github.com/crowdsecurity/go-cs-lib/pkg/ptr" - "github.com/crowdsecurity/go-cs-lib/pkg/slicetools" - - "github.com/crowdsecurity/crowdsec/pkg/models" - "github.com/crowdsecurity/crowdsec/pkg/types" -) - -// decisionRaw is only used to unmarshall json/csv decisions -type decisionRaw struct { - Duration string `csv:"duration,omitempty" json:"duration,omitempty"` - Scenario string `csv:"reason,omitempty" json:"reason,omitempty"` - Scope string `csv:"scope,omitempty" json:"scope,omitempty"` - Type string `csv:"type,omitempty" json:"type,omitempty"` - Value string `csv:"value" json:"value"` -} - -func parseDecisionList(content []byte, format string) ([]decisionRaw, error) { - ret := []decisionRaw{} - - switch format { - case "values": - log.Infof("Parsing values") - scanner := bufio.NewScanner(bytes.NewReader(content)) - for scanner.Scan() { - value := strings.TrimSpace(scanner.Text()) - ret = append(ret, decisionRaw{Value: value}) - } - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("unable to parse values: '%s'", err) - } - case "json": - log.Infof("Parsing json") - if err := json.Unmarshal(content, &ret); err != nil { - return nil, err - } - case "csv": - log.Infof("Parsing csv") - if err := csvutil.Unmarshal(content, &ret); err != nil { - return nil, fmt.Errorf("unable to parse csv: '%s'", err) - } - default: - return nil, fmt.Errorf("invalid format '%s', expected one of 'json', 'csv', 'values'", format) - } - - return ret, nil -} - - -func runDecisionsImport(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - input, err := flags.GetString("input") - if err != nil { - return err - } - - defaultDuration, err := flags.GetString("duration") - if err != nil { - return err - } - if defaultDuration == "" { - return fmt.Errorf("--duration cannot be empty") - } - - defaultScope, err := flags.GetString("scope") - if err != nil { - return err - } - if defaultScope == "" { - return fmt.Errorf("--scope cannot be empty") - } - - defaultReason, err := flags.GetString("reason") - if err != nil { - return err - } - if defaultReason == "" { - return fmt.Errorf("--reason cannot be empty") - } - - defaultType, err := flags.GetString("type") - if err != nil { - return err - } - if defaultType == "" { - return fmt.Errorf("--type cannot be empty") - } - - batchSize, err := flags.GetInt("batch") - if err != nil { - return err - } - - format, err := flags.GetString("format") - if err != nil { - return err - } - - var ( - content []byte - fin *os.File - ) - - // set format if the file has a json or csv extension - if format == "" { - if strings.HasSuffix(input, ".json") { - format = "json" - } else if strings.HasSuffix(input, ".csv") { - format = "csv" - } - } - - if format == "" { - return fmt.Errorf("unable to guess format from file extension, please provide a format with --format flag") - } - - if input == "-" { - fin = os.Stdin - input = "stdin" - } else { - fin, err = os.Open(input) - if err != nil { - return fmt.Errorf("unable to open %s: %s", input, err) - } - } - - content, err = io.ReadAll(fin) - if err != nil { - return fmt.Errorf("unable to read from %s: %s", input, err) - } - - decisionsListRaw, err := parseDecisionList(content, format) - if err != nil { - return err - } - - decisions := make([]*models.Decision, len(decisionsListRaw)) - for i, d := range decisionsListRaw { - if d.Value == "" { - return fmt.Errorf("item %d: missing 'value'", i) - } - - if d.Duration == "" { - d.Duration = defaultDuration - log.Debugf("item %d: missing 'duration', using default '%s'", i, defaultDuration) - } - - if d.Scenario == "" { - d.Scenario = defaultReason - log.Debugf("item %d: missing 'reason', using default '%s'", i, defaultReason) - } - - if d.Type == "" { - d.Type = defaultType - log.Debugf("item %d: missing 'type', using default '%s'", i, defaultType) - } - - if d.Scope == "" { - d.Scope = defaultScope - log.Debugf("item %d: missing 'scope', using default '%s'", i, defaultScope) - } - - decisions[i] = &models.Decision{ - Value: ptr.Of(d.Value), - Duration: ptr.Of(d.Duration), - Origin: ptr.Of(types.CscliImportOrigin), - Scenario: ptr.Of(d.Scenario), - Type: ptr.Of(d.Type), - Scope: ptr.Of(d.Scope), - Simulated: ptr.Of(false), - } - } - - alerts := models.AddAlertsRequest{} - - for _, chunk := range slicetools.Chunks(decisions, batchSize) { - log.Debugf("Processing chunk of %d decisions", len(chunk)) - importAlert := models.Alert{ - CreatedAt: time.Now().UTC().Format(time.RFC3339), - Scenario: ptr.Of(fmt.Sprintf("import %s: %d IPs", input, len(chunk))), - - Message: ptr.Of(""), - Events: []*models.Event{}, - Source: &models.Source{ - Scope: ptr.Of(""), - Value: ptr.Of(""), - }, - StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), - StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), - Capacity: ptr.Of(int32(0)), - Simulated: ptr.Of(false), - EventsCount: ptr.Of(int32(len(chunk))), - Leakspeed: ptr.Of(""), - ScenarioHash: ptr.Of(""), - ScenarioVersion: ptr.Of(""), - Decisions: chunk, - } - alerts = append(alerts, &importAlert) - } - - if len(decisions) > 1000 { - log.Infof("You are about to add %d decisions, this may take a while", len(decisions)) - } - - _, _, err = Client.Alerts.Add(context.Background(), alerts) - if err != nil { - return err - } - - log.Infof("Imported %d decisions", len(decisions)) - return nil -} - - -func NewDecisionsImportCmd() *cobra.Command { - var cmdDecisionsImport = &cobra.Command{ - Use: "import [options]", - Short: "Import decisions from a file or pipe", - Long: "expected format:\n" + - "csv : any of duration,reason,scope,type,value, with a header line\n" + - `json : {"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"}`, - DisableAutoGenTag: true, - Example: `decisions.csv: -duration,scope,value -24h,ip,1.2.3.4 - -$ cscli decisions import -i decisions.csv - -decisions.json: -[{"duration" : "4h", "scope" : "ip", "type" : "ban", "value" : "1.2.3.4"}] - -The file format is detected from the extension, but can be forced with the --format option -which is required when reading from standard input. - -Raw values, standard input: - -$ echo "1.2.3.4" | cscli decisions import -i - --format values -`, - RunE: runDecisionsImport, - } - - flags := cmdDecisionsImport.Flags() - flags.SortFlags = false - flags.StringP("input", "i", "", "Input file") - flags.StringP("duration", "d", "4h", "Decision duration: 1h,4h,30m") - flags.String("scope", types.Ip, "Decision scope: ip,range,username") - flags.StringP("reason", "R", "manual", "Decision reason: ") - flags.StringP("type", "t", "ban", "Decision type: ban,captcha,throttle") - flags.Int("batch", 0, "Split import in batches of N decisions") - flags.String("format", "", "Input format: 'json', 'csv' or 'values' (each line is a value, no headers)") - - cmdDecisionsImport.MarkFlagRequired("input") - - return cmdDecisionsImport -} diff --git a/cmd/crowdsec-cli/lapi.go b/cmd/crowdsec-cli/lapi.go index d03662c48..e8a29d9cd 100644 --- a/cmd/crowdsec-cli/lapi.go +++ b/cmd/crowdsec-cli/lapi.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/go-openapi/strfmt" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "golang.org/x/exp/slices" @@ -23,6 +24,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/parser" + "github.com/crowdsecurity/crowdsec/pkg/types" ) var LAPIURLPrefix string = "v1" @@ -203,7 +205,7 @@ func NewLapiCmd() *cobra.Command { DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { if err := csConfig.LoadAPIClient(); err != nil { - return fmt.Errorf("loading api client: %w", err) + return errors.Wrap(err, "loading api client") } return nil }, @@ -438,7 +440,7 @@ cscli lapi context delete --value evt.Line.Src return cmdContext } -func detectStaticField(GrokStatics []parser.ExtraField) []string { +func detectStaticField(GrokStatics []types.ExtraField) []string { ret := make([]string, 0) for _, static := range GrokStatics { if static.Parsed != "" { diff --git a/cmd/crowdsec-cli/machines.go b/cmd/crowdsec-cli/machines.go index 215943102..25bd5acec 100644 --- a/cmd/crowdsec-cli/machines.go +++ b/cmd/crowdsec-cli/machines.go @@ -12,6 +12,7 @@ import ( "time" "github.com/AlecAivazis/survey/v2" + "github.com/enescakir/emoji" "github.com/fatih/color" "github.com/go-openapi/strfmt" "github.com/google/uuid" @@ -84,21 +85,22 @@ func generateID(prefix string) (string, error) { return prefix + suffix, nil } -// getLastHeartbeat returns the last heartbeat timestamp of a machine -// and a boolean indicating if the machine is considered active or not. -func getLastHeartbeat(m *ent.Machine) (string, bool) { - if m.LastHeartbeat == nil { - return "-", false +func displayLastHeartBeat(m *ent.Machine, fancy bool) string { + var hbDisplay string + + if m.LastHeartbeat != nil { + lastHeartBeat := time.Now().UTC().Sub(*m.LastHeartbeat) + hbDisplay = lastHeartBeat.Truncate(time.Second).String() + if fancy && lastHeartBeat > 2*time.Minute { + hbDisplay = fmt.Sprintf("%s %s", emoji.Warning.String(), lastHeartBeat.Truncate(time.Second).String()) + } + } else { + hbDisplay = "-" + if fancy { + hbDisplay = emoji.Warning.String() + " -" + } } - - elapsed := time.Now().UTC().Sub(*m.LastHeartbeat) - - hb := elapsed.Truncate(time.Second).String() - if elapsed > 2*time.Minute { - return hb, false - } - - return hb, true + return hbDisplay } func getAgents(out io.Writer, dbClient *database.Client) error { @@ -128,10 +130,9 @@ func getAgents(out io.Writer, dbClient *database.Client) error { } else { validated = "false" } - hb, _ := getLastHeartbeat(m) - err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb}) + err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, displayLastHeartBeat(m, false)}) if err != nil { - return fmt.Errorf("failed to write raw output: %w", err) + return fmt.Errorf("failed to write raw output : %s", err) } } csvwriter.Flush() diff --git a/cmd/crowdsec-cli/machines_table.go b/cmd/crowdsec-cli/machines_table.go index e166fb785..cc15bb51b 100644 --- a/cmd/crowdsec-cli/machines_table.go +++ b/cmd/crowdsec-cli/machines_table.go @@ -24,11 +24,7 @@ func getAgentsTable(out io.Writer, machines []*ent.Machine) { validated = emoji.Prohibited.String() } - hb, active := getLastHeartbeat(m) - if !active { - hb = emoji.Warning.String() + " " + hb - } - t.AddRow(m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb) + t.AddRow(m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, displayLastHeartBeat(m, true)) } t.Render() diff --git a/cmd/crowdsec-cli/notifications.go b/cmd/crowdsec-cli/notifications.go index fe4c14f27..ebbefd6b7 100644 --- a/cmd/crowdsec-cli/notifications.go +++ b/cmd/crowdsec-cli/notifications.go @@ -15,6 +15,7 @@ import ( "github.com/fatih/color" "github.com/go-openapi/strfmt" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "gopkg.in/tomb.v2" @@ -27,12 +28,14 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csprofiles" ) + type NotificationsCfg struct { Config csplugin.PluginConfig `json:"plugin_config"` Profiles []*csconfig.ProfileCfg `json:"associated_profiles"` ids []uint } + func NewNotificationsCmd() *cobra.Command { var cmdNotifications = &cobra.Command{ Use: "notifications [action]", @@ -54,6 +57,7 @@ func NewNotificationsCmd() *cobra.Command { }, } + cmdNotifications.AddCommand(NewNotificationsListCmd()) cmdNotifications.AddCommand(NewNotificationsInspectCmd()) cmdNotifications.AddCommand(NewNotificationsReinjectCmd()) @@ -61,17 +65,18 @@ func NewNotificationsCmd() *cobra.Command { return cmdNotifications } + func getNotificationsConfiguration() (map[string]NotificationsCfg, error) { pcfgs := map[string]csplugin.PluginConfig{} wf := func(path string, info fs.FileInfo, err error) error { if info == nil { - return fmt.Errorf("error while traversing directory %s: %w", path, err) + return errors.Wrapf(err, "error while traversing directory %s", path) } name := filepath.Join(csConfig.ConfigPaths.NotificationDir, info.Name()) //Avoid calling info.Name() twice if (strings.HasSuffix(name, "yaml") || strings.HasSuffix(name, "yml")) && !(info.IsDir()) { ts, err := csplugin.ParsePluginConfigFile(name) if err != nil { - return fmt.Errorf("loading notifification plugin configuration with %s: %w", name, err) + return errors.Wrapf(err, "Loading notifification plugin configuration with %s", name) } for _, t := range ts { pcfgs[t.Name] = t @@ -81,14 +86,14 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) { } if err := filepath.Walk(csConfig.ConfigPaths.NotificationDir, wf); err != nil { - return nil, fmt.Errorf("while loading notifification plugin configuration: %w", err) + return nil, errors.Wrap(err, "Loading notifification plugin configuration") } // A bit of a tricky stuf now: reconcile profiles and notification plugins ncfgs := map[string]NotificationsCfg{} profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles) if err != nil { - return nil, fmt.Errorf("while extracting profiles from configuration: %w", err) + return nil, errors.Wrap(err, "Cannot extract profiles from configuration") } for profileID, profile := range profiles { loop: @@ -124,6 +129,7 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) { return ncfgs, nil } + func NewNotificationsListCmd() *cobra.Command { var cmdNotificationsList = &cobra.Command{ Use: "list", @@ -135,7 +141,7 @@ func NewNotificationsListCmd() *cobra.Command { RunE: func(cmd *cobra.Command, arg []string) error { ncfgs, err := getNotificationsConfiguration() if err != nil { - return fmt.Errorf("can't build profiles configuration: %w", err) + return errors.Wrap(err, "Can't build profiles configuration") } if csConfig.Cscli.Output == "human" { @@ -143,14 +149,14 @@ func NewNotificationsListCmd() *cobra.Command { } else if csConfig.Cscli.Output == "json" { x, err := json.MarshalIndent(ncfgs, "", " ") if err != nil { - return fmt.Errorf("failed to marshal notification configuration: %w", err) + return errors.New("failed to marshal notification configuration") } fmt.Printf("%s", string(x)) } else if csConfig.Cscli.Output == "raw" { csvwriter := csv.NewWriter(os.Stdout) err := csvwriter.Write([]string{"Name", "Type", "Profile name"}) if err != nil { - return fmt.Errorf("failed to write raw header: %w", err) + return errors.Wrap(err, "failed to write raw header") } for _, b := range ncfgs { profilesList := []string{} @@ -159,7 +165,7 @@ func NewNotificationsListCmd() *cobra.Command { } err := csvwriter.Write([]string{b.Config.Name, b.Config.Type, strings.Join(profilesList, ", ")}) if err != nil { - return fmt.Errorf("failed to write raw content: %w", err) + return errors.Wrap(err, "failed to write raw content") } } csvwriter.Flush() @@ -171,6 +177,7 @@ func NewNotificationsListCmd() *cobra.Command { return cmdNotificationsList } + func NewNotificationsInspectCmd() *cobra.Command { var cmdNotificationsInspect = &cobra.Command{ Use: "inspect", @@ -188,14 +195,14 @@ func NewNotificationsInspectCmd() *cobra.Command { pluginName := arg[0] if pluginName == "" { - return fmt.Errorf("please provide a plugin name to inspect") + errors.New("Please provide a plugin name to inspect") } ncfgs, err := getNotificationsConfiguration() if err != nil { - return fmt.Errorf("can't build profiles configuration: %w", err) + return errors.Wrap(err, "Can't build profiles configuration") } if cfg, ok = ncfgs[pluginName]; !ok { - return fmt.Errorf("plugin '%s' does not exist or is not active", pluginName) + return errors.New("The provided plugin name doesn't exist or isn't active") } if csConfig.Cscli.Output == "human" || csConfig.Cscli.Output == "raw" { @@ -209,7 +216,7 @@ func NewNotificationsInspectCmd() *cobra.Command { } else if csConfig.Cscli.Output == "json" { x, err := json.MarshalIndent(cfg, "", " ") if err != nil { - return fmt.Errorf("failed to marshal notification configuration: %w", err) + return errors.New("failed to marshal notification configuration") } fmt.Printf("%s", string(x)) } @@ -220,6 +227,7 @@ func NewNotificationsInspectCmd() *cobra.Command { return cmdNotificationsInspect } + func NewNotificationsReinjectCmd() *cobra.Command { var remediation bool var alertOverride string @@ -242,26 +250,26 @@ cscli notifications reinject -a '{"remediation": true,"scenario":"not ) if len(args) != 1 { printHelp(cmd) - return fmt.Errorf("wrong number of argument: there should be one argument") + return errors.New("Wrong number of argument: there should be one argument") } //first: get the alert id, err := strconv.Atoi(args[0]) if err != nil { - return fmt.Errorf("bad alert id %s", args[0]) + return errors.New(fmt.Sprintf("bad alert id %s", args[0])) } if err := csConfig.LoadAPIClient(); err != nil { - return fmt.Errorf("loading api client: %w", err) + return errors.Wrapf(err, "loading api client") } if csConfig.API.Client == nil { - return fmt.Errorf("missing configuration on 'api_client:'") + return errors.New("There is no configuration on 'api_client:'") } if csConfig.API.Client.Credentials == nil { - return fmt.Errorf("missing API credentials in '%s'", csConfig.API.Client.CredentialsFilePath) + return errors.New(fmt.Sprintf("Please provide credentials for the API in '%s'", csConfig.API.Client.CredentialsFilePath)) } apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL) if err != nil { - return fmt.Errorf("error parsing the URL of the API: %w", err) + return errors.Wrapf(err, "error parsing the URL of the API") } client, err := apiclient.NewClient(&apiclient.Config{ MachineID: csConfig.API.Client.Credentials.Login, @@ -271,16 +279,16 @@ cscli notifications reinject -a '{"remediation": true,"scenario":"not VersionPrefix: "v1", }) if err != nil { - return fmt.Errorf("error creating the client for the API: %w", err) + return errors.Wrapf(err, "error creating the client for the API") } alert, _, err := client.Alerts.GetByID(context.Background(), id) if err != nil { - return fmt.Errorf("can't find alert with id %s: %w", args[0], err) + return errors.Wrapf(err, fmt.Sprintf("can't find alert with id %s", args[0])) } if alertOverride != "" { if err = json.Unmarshal([]byte(alertOverride), alert); err != nil { - return fmt.Errorf("can't unmarshal data in the alert flag: %w", err) + return errors.Wrapf(err, "Can't unmarshal the data given in the alert flag") } } if !remediation { @@ -290,7 +298,7 @@ cscli notifications reinject -a '{"remediation": true,"scenario":"not // second we start plugins err = pluginBroker.Init(csConfig.PluginConfig, csConfig.API.Server.Profiles, csConfig.ConfigPaths) if err != nil { - return fmt.Errorf("can't initialize plugins: %w", err) + return errors.Wrapf(err, "Can't initialize plugins") } pluginTomb.Go(func() error { @@ -302,13 +310,13 @@ cscli notifications reinject -a '{"remediation": true,"scenario":"not profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles) if err != nil { - return fmt.Errorf("cannot extract profiles from configuration: %w", err) + return errors.Wrap(err, "Cannot extract profiles from configuration") } for id, profile := range profiles { _, matched, err := profile.EvaluateProfile(alert) if err != nil { - return fmt.Errorf("can't evaluate profile %s: %w", profile.Cfg.Name, err) + return errors.Wrapf(err, "can't evaluate profile %s", profile.Cfg.Name) } if !matched { log.Infof("The profile %s didn't match", profile.Cfg.Name) @@ -336,7 +344,7 @@ cscli notifications reinject -a '{"remediation": true,"scenario":"not } // time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent - pluginTomb.Kill(fmt.Errorf("terminating")) + pluginTomb.Kill(errors.New("terminating")) pluginTomb.Wait() return nil }, diff --git a/cmd/crowdsec-cli/scenarios.go b/cmd/crowdsec-cli/scenarios.go index de52dcb48..a5b433228 100644 --- a/cmd/crowdsec-cli/scenarios.go +++ b/cmd/crowdsec-cli/scenarios.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/fatih/color" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -32,7 +33,7 @@ cscli scenarios remove crowdsecurity/ssh-bf } if err := cwhub.SetHubBranch(); err != nil { - return fmt.Errorf("while setting hub branch: %w", err) + return errors.Wrap(err, "while setting hub branch") } if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go index 66c1493a4..013abf4b2 100644 --- a/cmd/crowdsec-cli/support.go +++ b/cmd/crowdsec-cli/support.go @@ -26,6 +26,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/fflag" "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/crowdsecurity/crowdsec/pkg/types" ) const ( @@ -47,14 +48,6 @@ const ( SUPPORT_CROWDSEC_PROFILE_PATH = "config/profiles.yaml" ) -// from https://github.com/acarl005/stripansi -var reStripAnsi = regexp.MustCompile("[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))") - -func stripAnsiString(str string) string { - // the byte version doesn't strip correctly - return reStripAnsi.ReplaceAllString(str, "") -} - func collectMetrics() ([]byte, []byte, error) { log.Info("Collecting prometheus metrics") err := csConfig.LoadPrometheus() @@ -407,7 +400,7 @@ cscli support dump -f /tmp/crowdsec-support.zip log.Errorf("Could not add zip entry for %s: %s", filename, err) continue } - fw.Write([]byte(stripAnsiString(string(data)))) + fw.Write([]byte(types.StripAnsiString(string(data)))) } err = zipWriter.Close() diff --git a/cmd/crowdsec-cli/utils.go b/cmd/crowdsec-cli/utils.go index f77342cc9..1a99c2acf 100644 --- a/cmd/crowdsec-cli/utils.go +++ b/cmd/crowdsec-cli/utils.go @@ -598,7 +598,7 @@ func RestoreHub(dirPath string) error { log.Infof("Going to restore local/tainted [%s]", tfile.Name()) sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name()) destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name()) - if err = CopyFile(sourceFile, destinationFile); err != nil { + if err = types.CopyFile(sourceFile, destinationFile); err != nil { return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err) } log.Infof("restored %s to %s", sourceFile, destinationFile) @@ -607,7 +607,7 @@ func RestoreHub(dirPath string) error { log.Infof("Going to restore local/tainted [%s]", file.Name()) sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name()) destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name()) - if err = CopyFile(sourceFile, destinationFile); err != nil { + if err = types.CopyFile(sourceFile, destinationFile); err != nil { return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err) } log.Infof("restored %s to %s", sourceFile, destinationFile) @@ -657,7 +657,7 @@ func BackupHub(dirPath string) error { } clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate) tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName) - if err = CopyFile(v.LocalPath, tfile); err != nil { + if err = types.CopyFile(v.LocalPath, tfile); err != nil { return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err) } clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile) diff --git a/cmd/crowdsec/api.go b/cmd/crowdsec/api.go index fd2e2ce08..3ce249d4c 100644 --- a/cmd/crowdsec/api.go +++ b/cmd/crowdsec/api.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "runtime" "time" @@ -21,7 +20,7 @@ func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) { apiServer, err := apiserver.NewServer(cConfig.API.Server) if err != nil { - return nil, fmt.Errorf("unable to run local API: %w", err) + return nil, errors.Wrap(err, "unable to run local API") } if hasPlugins(cConfig.API.Server.Profiles) { @@ -30,27 +29,23 @@ func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) { if cConfig.PluginConfig == nil && runtime.GOOS != "windows" { return nil, errors.New("plugins are enabled, but the plugin_config section is missing in the configuration") } - if cConfig.ConfigPaths.NotificationDir == "" { return nil, errors.New("plugins are enabled, but config_paths.notification_dir is not defined") } - if cConfig.ConfigPaths.PluginDir == "" { return nil, errors.New("plugins are enabled, but config_paths.plugin_dir is not defined") } - err = pluginBroker.Init(cConfig.PluginConfig, cConfig.API.Server.Profiles, cConfig.ConfigPaths) if err != nil { - return nil, fmt.Errorf("unable to run plugin broker: %w", err) + return nil, errors.Wrap(err, "unable to run local API") } - log.Info("initiated plugin broker") apiServer.AttachPluginBroker(&pluginBroker) } err = apiServer.InitController() if err != nil { - return nil, fmt.Errorf("unable to run local API: %w", err) + return nil, errors.Wrap(err, "unable to run local API") } return apiServer, nil diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index 68a7c6180..8b4487e15 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -3,12 +3,10 @@ package main import ( "fmt" "os" - "path/filepath" "sync" "time" - log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" + "path/filepath" "github.com/crowdsecurity/go-cs-lib/pkg/trace" @@ -18,28 +16,31 @@ import ( leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket" "github.com/crowdsecurity/crowdsec/pkg/parser" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" ) func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) { var err error // Populate cwhub package tools - if err = cwhub.GetHubIdx(cConfig.Hub); err != nil { - return nil, fmt.Errorf("while loading hub index: %w", err) + if err := cwhub.GetHubIdx(cConfig.Hub); err != nil { + return &parser.Parsers{}, fmt.Errorf("Failed to load hub index : %s", err) } // Start loading configs csParsers := parser.NewParsers() if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil { - return nil, fmt.Errorf("while loading parsers: %w", err) + return &parser.Parsers{}, fmt.Errorf("Failed to load parsers: %s", err) } if err := LoadBuckets(cConfig); err != nil { - return nil, fmt.Errorf("while loading scenarios: %w", err) + return &parser.Parsers{}, fmt.Errorf("Failed to load scenarios: %s", err) } if err := LoadAcquisition(cConfig); err != nil { - return nil, fmt.Errorf("while loading acquisition config: %w", err) + return &parser.Parsers{}, fmt.Errorf("Error while loading acquisition config : %s", err) } return csParsers, nil } @@ -117,7 +118,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error { aggregated = true } if err := acquisition.GetMetrics(dataSources, aggregated); err != nil { - return fmt.Errorf("while fetching prometheus metrics for datasources: %w", err) + return errors.Wrap(err, "while fetching prometheus metrics for datasources.") } } diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index 0de6f0378..767097f0e 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -51,15 +51,12 @@ var ( ) type Flags struct { - ConfigFile string - - LogLevelTrace bool - LogLevelDebug bool - LogLevelInfo bool - LogLevelWarn bool - LogLevelError bool - LogLevelFatal bool - + ConfigFile string + TraceLevel bool + DebugLevel bool + InfoLevel bool + WarnLevel bool + ErrorLevel bool PrintVersion bool SingleFileType string Labels map[string]string @@ -110,7 +107,7 @@ func LoadAcquisition(cConfig *csconfig.Config) error { dataSources, err = acquisition.LoadAcquisitionFromDSN(flags.OneShotDSN, flags.Labels, flags.Transform) if err != nil { - return fmt.Errorf("failed to configure datasource for %s: %w", flags.OneShotDSN, err) + return errors.Wrapf(err, "failed to configure datasource for %s", flags.OneShotDSN) } } else { dataSources, err = acquisition.LoadAcquisitionFromFile(cConfig.Crowdsec) @@ -119,10 +116,6 @@ func LoadAcquisition(cConfig *csconfig.Config) error { } } - if len(dataSources) == 0 { - return fmt.Errorf("no datasource enabled") - } - return nil } @@ -147,14 +140,11 @@ func (l labelsMap) Set(label string) error { func (f *Flags) Parse() { flag.StringVar(&f.ConfigFile, "c", csconfig.DefaultConfigPath("config.yaml"), "configuration file") - - flag.BoolVar(&f.LogLevelTrace, "trace", false, "set log level to 'trace' (VERY verbose)") - flag.BoolVar(&f.LogLevelDebug, "debug", false, "set log level to 'debug'") - flag.BoolVar(&f.LogLevelInfo, "info", false, "set log level to 'info'") - flag.BoolVar(&f.LogLevelWarn, "warning", false, "set log level to 'warning'") - flag.BoolVar(&f.LogLevelError, "error", false, "set log level to 'error'") - flag.BoolVar(&f.LogLevelFatal, "fatal", false, "set log level to 'fatal'") - + flag.BoolVar(&f.TraceLevel, "trace", false, "VERY verbose") + flag.BoolVar(&f.DebugLevel, "debug", false, "print debug-level on stderr") + flag.BoolVar(&f.InfoLevel, "info", false, "print info-level on stderr") + flag.BoolVar(&f.WarnLevel, "warning", false, "print warning-level on stderr") + flag.BoolVar(&f.ErrorLevel, "error", false, "print error-level on stderr") flag.BoolVar(&f.PrintVersion, "version", false, "display version") flag.StringVar(&f.OneShotDSN, "dsn", "", "Process a single data source in time-machine") flag.StringVar(&f.Transform, "transform", "", "expr to apply on the event after acquisition") @@ -182,18 +172,16 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level { // override from flags switch { - case f.LogLevelTrace: + case f.TraceLevel: ret = log.TraceLevel - case f.LogLevelDebug: + case f.DebugLevel: ret = log.DebugLevel - case f.LogLevelInfo: + case f.InfoLevel: ret = log.InfoLevel - case f.LogLevelWarn: + case f.WarnLevel: ret = log.WarnLevel - case f.LogLevelError: + case f.ErrorLevel: ret = log.ErrorLevel - case f.LogLevelFatal: - ret = log.FatalLevel default: } diff --git a/cmd/crowdsec/output.go b/cmd/crowdsec/output.go index 67489f459..17cc99827 100644 --- a/cmd/crowdsec/output.go +++ b/cmd/crowdsec/output.go @@ -7,10 +7,6 @@ import ( "sync" "time" - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/go-cs-lib/pkg/version" "github.com/crowdsecurity/crowdsec/pkg/apiclient" @@ -20,6 +16,9 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/parser" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/go-openapi/strfmt" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) func dedupAlerts(alerts []types.RuntimeAlert) ([]*models.Alert, error) { @@ -51,11 +50,11 @@ func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error alertsToPush, err := dedupAlerts(alerts) if err != nil { - return fmt.Errorf("failed to transform alerts for api: %w", err) + return errors.Wrap(err, "failed to transform alerts for api") } _, _, err = client.Alerts.Add(ctx, alertsToPush) if err != nil { - return fmt.Errorf("failed sending alert to LAPI: %w", err) + return errors.Wrap(err, "failed sending alert to LAPI") } return nil } @@ -105,11 +104,11 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky Scenarios: scenarios, }) if err != nil { - return fmt.Errorf("authenticate watcher (%s): %w", apiConfig.Login, err) + return errors.Wrapf(err, "authenticate watcher (%s)", apiConfig.Login) } if err := Client.GetClient().Transport.(*apiclient.JWTTransport).Expiration.UnmarshalText([]byte(authResp.Expire)); err != nil { - return fmt.Errorf("unable to parse jwt expiration: %w", err) + return errors.Wrap(err, "unable to parse jwt expiration") } Client.GetClient().Transport.(*apiclient.JWTTransport).Token = authResp.Token diff --git a/cmd/crowdsec/pour.go b/cmd/crowdsec/pour.go index 3f717e397..adb072376 100644 --- a/cmd/crowdsec/pour.go +++ b/cmd/crowdsec/pour.go @@ -12,7 +12,9 @@ import ( ) func runPour(input chan types.Event, holders []leaky.BucketFactory, buckets *leaky.Buckets, cConfig *csconfig.Config) error { - count := 0 + var ( + count int + ) for { //bucket is now ready select { diff --git a/cmd/crowdsec/run_in_svc_windows.go b/cmd/crowdsec/run_in_svc_windows.go index d63a587ac..c51d24147 100644 --- a/cmd/crowdsec/run_in_svc_windows.go +++ b/cmd/crowdsec/run_in_svc_windows.go @@ -3,6 +3,7 @@ package main import ( "fmt" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/sys/windows/svc" @@ -21,7 +22,7 @@ func StartRunSvc() error { isRunninginService, err := svc.IsWindowsService() if err != nil { - return fmt.Errorf("failed to determine if we are running in windows service mode: %w", err) + return errors.Wrap(err, "failed to determine if we are running in windows service mode") } if isRunninginService { return runService(svcName) @@ -30,22 +31,22 @@ func StartRunSvc() error { if flags.WinSvc == "Install" { err = installService(svcName, svcDescription) if err != nil { - return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err) + return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName) } } else if flags.WinSvc == "Remove" { err = removeService(svcName) if err != nil { - return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err) + return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName) } } else if flags.WinSvc == "Start" { err = startService(svcName) if err != nil { - return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err) + return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName) } } else if flags.WinSvc == "Stop" { err = controlService(svcName, svc.Stop, svc.Stopped) if err != nil { - return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err) + return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName) } } else if flags.WinSvc == "" { return WindowsRun() @@ -65,7 +66,7 @@ func WindowsRun() error { if err != nil { return err } - + // Configure logging log.Infof("Crowdsec %s", version.String()) apiReady := make(chan bool, 1) diff --git a/cmd/crowdsec/serve.go b/cmd/crowdsec/serve.go index 5d365b410..5e2e8b720 100644 --- a/cmd/crowdsec/serve.go +++ b/cmd/crowdsec/serve.go @@ -1,16 +1,16 @@ package main import ( - "fmt" "os" "os/signal" "syscall" "time" + "github.com/coreos/go-systemd/v22/daemon" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" - "github.com/crowdsecurity/go-cs-lib/pkg/csdaemon" "github.com/crowdsecurity/go-cs-lib/pkg/trace" "github.com/crowdsecurity/crowdsec/pkg/csconfig" @@ -68,7 +68,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) { } apiServer, err := initAPIServer(cConfig) if err != nil { - return nil, fmt.Errorf("unable to init api server: %w", err) + return nil, errors.Wrap(err, "unable to init api server") } apiReady := make(chan bool, 1) @@ -78,7 +78,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) { if !cConfig.DisableAgent { csParsers, err := initCrowdsec(cConfig) if err != nil { - return nil, fmt.Errorf("unable to init crowdsec: %w", err) + return nil, errors.Wrap(err, "unable to init crowdsec") } // restore bucket state @@ -180,13 +180,13 @@ func shutdownCrowdsec() error { func shutdown(sig os.Signal, cConfig *csconfig.Config) error { if !cConfig.DisableAgent { if err := shutdownCrowdsec(); err != nil { - return fmt.Errorf("failed to shut down crowdsec: %w", err) + return errors.Wrap(err, "failed to shut down crowdsec") } } if !cConfig.DisableAPI { if err := shutdownAPI(); err != nil { - return fmt.Errorf("failed to shut down api routines: %w", err) + return errors.Wrap(err, "failed to shut down api routines") } } @@ -238,13 +238,13 @@ func HandleSignals(cConfig *csconfig.Config) error { log.Warning("SIGHUP received, reloading") if err = shutdown(s, cConfig); err != nil { - exitChan <- fmt.Errorf("failed shutdown: %w", err) + exitChan <- errors.Wrap(err, "failed shutdown") break Loop } if newConfig, err = reloadHandler(s); err != nil { - exitChan <- fmt.Errorf("reload handler failure: %w", err) + exitChan <- errors.Wrap(err, "reload handler failure") break Loop } @@ -256,7 +256,7 @@ func HandleSignals(cConfig *csconfig.Config) error { case os.Interrupt, syscall.SIGTERM: log.Warning("SIGTERM received, shutting down") if err = shutdown(s, cConfig); err != nil { - exitChan <- fmt.Errorf("failed shutdown: %w", err) + exitChan <- errors.Wrap(err, "failed shutdown") break Loop } @@ -284,17 +284,17 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e if cConfig.API.Server != nil && cConfig.API.Server.DbConfig != nil { dbClient, err := database.NewClient(cConfig.API.Server.DbConfig) if err != nil { - return fmt.Errorf("failed to get database client: %w", err) + return errors.Wrap(err, "failed to get database client") } err = exprhelpers.Init(dbClient) if err != nil { - return fmt.Errorf("failed to init expr helpers: %w", err) + return errors.Wrap(err, "failed to init expr helpers") } } else { err := exprhelpers.Init(nil) if err != nil { - return fmt.Errorf("failed to init expr helpers: %w", err) + return errors.Wrap(err, "failed to init expr helpers") } log.Warningln("Exprhelpers loaded without database client.") @@ -303,7 +303,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e if cConfig.API.CTI != nil && *cConfig.API.CTI.Enabled { log.Infof("Crowdsec CTI helper enabled") if err := exprhelpers.InitCrowdsecCTI(cConfig.API.CTI.Key, cConfig.API.CTI.CacheTimeout, cConfig.API.CTI.CacheSize, cConfig.API.CTI.LogLevel); err != nil { - return fmt.Errorf("failed to init crowdsec cti: %w", err) + return errors.Wrap(err, "failed to init crowdsec cti") } } @@ -319,7 +319,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e apiServer, err := initAPIServer(cConfig) if err != nil { - return fmt.Errorf("api server init: %w", err) + return errors.Wrap(err, "api server init") } if !flags.TestMode { @@ -332,7 +332,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e if !cConfig.DisableAgent { csParsers, err := initCrowdsec(cConfig) if err != nil { - return fmt.Errorf("crowdsec init: %w", err) + return errors.Wrap(err, "crowdsec init") } // if it's just linting, we're done @@ -350,7 +350,10 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e } if cConfig.Common != nil && cConfig.Common.Daemonize { - csdaemon.NotifySystemd(log.StandardLogger()) + sent, err := daemon.SdNotify(false, daemon.SdNotifyReady) + if !sent || err != nil { + log.Errorf("Failed to notify(sent: %v): %v", sent, err) + } // wait for signals return HandleSignals(cConfig) } diff --git a/cmd/crowdsec/win_service.go b/cmd/crowdsec/win_service.go index ab9ecc815..d0e80c58a 100644 --- a/cmd/crowdsec/win_service.go +++ b/cmd/crowdsec/win_service.go @@ -8,10 +8,10 @@ package main import ( - "fmt" "syscall" "time" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/sys/windows" "golang.org/x/sys/windows/svc" @@ -106,7 +106,7 @@ func runService(name string) error { winsvc := crowdsec_winservice{config: cConfig} if err := svc.Run(name, &winsvc); err != nil { - return fmt.Errorf("%s service failed: %w", name, err) + return errors.Wrapf(err, "%s service failed", name) } log.Infof("%s service stopped", name) diff --git a/debian/control b/debian/control index 4673284e7..d06d38884 100644 --- a/debian/control +++ b/debian/control @@ -1,8 +1,6 @@ Source: crowdsec Maintainer: Crowdsec Team -Build-Depends: debhelper, bash -Section: admin -Priority: optional +Build-Depends: debhelper, bash, git Package: crowdsec Architecture: any diff --git a/debian/rules b/debian/rules index e6202a6f7..6683e5443 100755 --- a/debian/rules +++ b/debian/rules @@ -1,6 +1,6 @@ #!/usr/bin/make -f -export DEB_VERSION=$(shell dpkg-parsechangelog | grep -E '^Version:' | cut -f 2 -d ' ') +export DEB_VERSION=$(shell dpkg-parsechangelog | egrep '^Version:' | cut -f 2 -d ' ') export BUILD_VERSION=v${DEB_VERSION}-debian-pragmatic export GO111MODULE=on @@ -11,10 +11,12 @@ override_dh_auto_clean: override_dh_auto_test: override_dh_auto_build: override_dh_auto_install: - - # just use the prebuilt binaries, otherwise: - # make build BUILD_RE_WASM=0 BUILD_STATIC=1 - +# mkdir /tmp/go +# echo $(go version) +# echo $($GOCMD version) +# cd cmd/crowdsec && GOROOT=/tmp/go GO111MODULE=on $(GOBUILD) $(LD_OPTS) -o $(CROWDSEC_BIN) -v && cd .. +# cd cmd/crowdsec-cli && GOROOT=/tmp/go GO111MODULE=on $(GOBUILD) $(LD_OPTS) -o cscli -v && cd .. + make build mkdir -p debian/crowdsec/usr/bin mkdir -p debian/crowdsec/etc/crowdsec mkdir -p debian/crowdsec/usr/share/crowdsec diff --git a/debian/templates b/debian/templates index c6998eb85..c07ef8446 100644 --- a/debian/templates +++ b/debian/templates @@ -17,7 +17,7 @@ Description: Address of the local API server Template: crowdsec/capi Type: boolean Default: true -Description: Do you want to use the centralized remote API server ? +Description: Do you want to the centralized remote API server ? To share information with other crowdsec you can register to the centralized remote API server. . If you don't know what to do, answer yes. diff --git a/docker/docker_start.sh b/docker/docker_start.sh index 21b42dcb0..8ec449103 100755 --- a/docker/docker_start.sh +++ b/docker/docker_start.sh @@ -56,7 +56,7 @@ conf_get() { if [ $# -ge 2 ]; then yq e "$1" "$2" else - cscli config show-yaml | yq e "$1" + yq e "$1" "$CONFIG_FILE" fi } diff --git a/docker/test/tests/test_hub_collections.py b/docker/test/tests/test_hub_collections.py index b890bebb9..81567954b 100644 --- a/docker/test/tests/test_hub_collections.py +++ b/docker/test/tests/test_hub_collections.py @@ -6,8 +6,11 @@ Test collection management from http import HTTPStatus import json +import os +import pwd import pytest +import yaml pytestmark = pytest.mark.docker @@ -82,7 +85,12 @@ def test_taint_bubble_up(crowdsec, tmp_path_factory, flavor): 'COLLECTIONS': f'{coll}' } - with crowdsec(flavor=flavor, environment=env) as cs: + hub = tmp_path_factory.mktemp("hub") + volumes = { + hub: {'bind': '/etc/crowdsec/hub', 'mode': 'rw'} + } + + with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs: cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK) res = cs.cont.exec_run('cscli collections list -o json') assert res.exit_code == 0 @@ -94,13 +102,25 @@ def test_taint_bubble_up(crowdsec, tmp_path_factory, flavor): f'*Enabled collections : {coll}*', ]) - scenario = 'crowdsecurity/http-crawl-non_statics' - - # the description won't be read back, it's from the index - yq_command = f"yq -e -i '.description=\"tainted\"' /etc/crowdsec/hub/scenarios/{scenario}.yaml" - res = cs.cont.exec_run(yq_command) + # change file permissions to allow edit + current_uid = pwd.getpwuid(os.getuid()).pw_uid + res = cs.cont.exec_run(f'chown -R {current_uid} /etc/crowdsec/hub') assert res.exit_code == 0 + scenario = 'crowdsecurity/http-crawl-non_statics' + scenario_file = hub / f'scenarios/{scenario}.yaml' + + with open(scenario_file) as f: + yml = yaml.safe_load(f) + + yml['description'] += ' (tainted)' + # won't be able to read it back because description is taken from the index + + with open(scenario_file, 'w') as f: + yaml.dump(yml, f) + + with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs: + cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK) res = cs.cont.exec_run(f'cscli scenarios inspect {scenario} -o json') assert res.exit_code == 0 j = json.loads(res.output) diff --git a/go.mod b/go.mod index f547709e8..d64e1c075 100644 --- a/go.mod +++ b/go.mod @@ -5,22 +5,14 @@ go 1.20 require ( entgo.io/ent v0.11.3 github.com/AlecAivazis/survey/v2 v2.2.7 - github.com/Masterminds/semver/v3 v3.1.1 - github.com/Masterminds/sprig/v3 v3.2.2 + github.com/Microsoft/go-winio v0.5.2 // indirect github.com/alexliesenfeld/health v0.5.1 github.com/antonmedv/expr v1.12.5 github.com/appleboy/gin-jwt/v2 v2.8.0 - github.com/aquasecurity/table v1.8.0 - github.com/aws/aws-lambda-go v1.38.0 github.com/aws/aws-sdk-go v1.42.25 - github.com/beevik/etree v1.1.0 - github.com/blackfireio/osinfo v1.0.3 - github.com/bluele/gcache v0.0.2 github.com/buger/jsonparser v1.1.1 github.com/c-robinson/iplib v1.0.3 - github.com/cespare/xxhash/v2 v2.1.2 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 - github.com/crowdsecurity/go-cs-lib v0.0.2 github.com/crowdsecurity/grokky v0.2.1 github.com/crowdsecurity/machineid v1.0.2 github.com/davecgh/go-spew v1.1.1 @@ -28,7 +20,7 @@ require ( github.com/docker/docker v20.10.24+incompatible github.com/docker/go-connections v0.4.0 github.com/enescakir/emoji v1.0.0 - github.com/fatih/color v1.15.0 + github.com/fatih/color v1.13.0 github.com/fsnotify/fsnotify v1.6.0 github.com/gin-gonic/gin v1.7.7 github.com/go-co-op/gocron v1.17.0 @@ -37,50 +29,65 @@ require ( github.com/go-openapi/swag v0.19.14 github.com/go-openapi/validate v0.20.0 github.com/go-sql-driver/mysql v1.6.0 - github.com/goccy/go-yaml v1.9.7 - github.com/gofrs/uuid v4.0.0+incompatible - github.com/golang-jwt/jwt/v4 v4.2.0 github.com/google/go-querystring v1.0.0 github.com/google/uuid v1.3.0 - github.com/google/winops v0.0.0-20211216095627-f0e86eb1453b github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e github.com/hashicorp/go-hclog v1.0.0 github.com/hashicorp/go-plugin v1.4.2 github.com/hashicorp/go-version v1.2.1 - github.com/ivanpirog/coloredcobra v1.0.1 github.com/jackc/pgx/v4 v4.14.1 github.com/jarcoal/httpmock v1.1.0 github.com/jszwec/csvutil v1.5.1 - github.com/lithammer/dedent v1.1.0 - github.com/mattn/go-isatty v0.0.17 github.com/mattn/go-sqlite3 v1.14.16 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 github.com/nxadm/tail v1.4.8 github.com/oschwald/geoip2-golang v1.4.0 github.com/oschwald/maxminddb-golang v1.8.0 - github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.14.0 github.com/prometheus/client_model v0.3.0 github.com/prometheus/prom2json v1.3.0 github.com/r3labs/diff/v2 v2.14.1 - github.com/segmentio/kafka-go v0.4.34 - github.com/shirou/gopsutil/v3 v3.23.5 github.com/sirupsen/logrus v1.9.2 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.3 - github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c - github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 - github.com/wasilibs/go-re2 v0.2.1 golang.org/x/crypto v0.1.0 - golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 - golang.org/x/mod v0.11.0 - golang.org/x/sys v0.9.0 + golang.org/x/mod v0.8.0 google.golang.org/grpc v1.47.0 google.golang.org/protobuf v1.28.1 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 gopkg.in/yaml.v2 v2.4.0 + gotest.tools/v3 v3.0.3 +) + +require ( + github.com/Masterminds/semver v1.5.0 + github.com/Masterminds/sprig/v3 v3.2.2 + github.com/aquasecurity/table v1.8.0 + github.com/aws/aws-lambda-go v1.38.0 + github.com/beevik/etree v1.1.0 + github.com/blackfireio/osinfo v1.0.3 + github.com/bluele/gcache v0.0.2 + github.com/cespare/xxhash/v2 v2.1.2 + github.com/corazawaf/coraza/v3 v3.0.0-00010101000000-000000000000 + github.com/coreos/go-systemd/v22 v22.5.0 + github.com/crowdsecurity/go-cs-lib v0.0.0-20230531105801-4c1535c2b3bd + github.com/goccy/go-yaml v1.9.7 + github.com/gofrs/uuid v4.0.0+incompatible + github.com/golang-jwt/jwt/v4 v4.2.0 + github.com/google/winops v0.0.0-20211216095627-f0e86eb1453b + github.com/ivanpirog/coloredcobra v1.0.1 + github.com/lithammer/dedent v1.1.0 + github.com/mattn/go-isatty v0.0.14 + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 + github.com/segmentio/kafka-go v0.4.34 + github.com/shirou/gopsutil/v3 v3.22.12 + github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c + github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 + github.com/wasilibs/go-re2 v0.2.1 + golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc + golang.org/x/sys v0.7.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/apiserver v0.22.5 ) @@ -88,7 +95,7 @@ require ( require ( ariga.io/atlas v0.7.2-0.20220927111110-867ee0cca56a // indirect github.com/Masterminds/goutils v1.1.1 // indirect - github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/Masterminds/semver/v3 v3.1.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/agext/levenshtein v1.2.1 // indirect @@ -96,7 +103,7 @@ require ( github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/corazawaf/libinjection-go v0.1.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/go-units v0.4.0 // indirect @@ -140,7 +147,7 @@ require ( github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magefile/mage v1.14.0 // indirect github.com/mailru/easyjson v0.7.6 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect @@ -155,6 +162,7 @@ require ( github.com/oklog/run v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect + github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect @@ -163,24 +171,24 @@ require ( github.com/rivo/uniseg v0.2.0 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/spf13/cast v1.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tetratelabs/wazero v1.0.0-rc.2 // indirect - github.com/tidwall/gjson v1.13.0 // indirect + github.com/tidwall/gjson v1.14.4 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect github.com/ugorji/go/codec v1.2.6 // indirect - github.com/vmihailenco/msgpack v4.0 .4+incompatible // indirect - github.com/yusufpapurcu/wmi v1.2.3 // indirect + github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect + github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/zclconf/go-cty v1.8.0 // indirect - go.mongodb.org/mongo-driver v1.9.4 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect - golang.org/x/tools v0.6.0 // indirect + go.mongodb.org/mongo-driver v1.9.0 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect + golang.org/x/term v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect @@ -190,8 +198,11 @@ require ( k8s.io/apimachinery v0.25.2 // indirect k8s.io/klog/v2 v2.70.1 // indirect k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect + rsc.io/binaryregexp v0.2.0 // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) replace golang.org/x/time/rate => github.com/crowdsecurity/crowdsec/pkg/time/rate v0.0.0 + +replace github.com/corazawaf/coraza/v3 => ./coraza diff --git a/go.sum b/go.sum index 41bff5d47..8d4506758 100644 --- a/go.sum +++ b/go.sum @@ -55,12 +55,14 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= -github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8 h1:xzYJEypr/85nBpB11F9br+3HUrpgb+fcm5iADzXXYEw= @@ -149,6 +151,8 @@ github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMe github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/corazawaf/libinjection-go v0.1.2 h1:oeiV9pc5rvJ+2oqOqXEAMJousPpGiup6f7Y3nZj5GoM= +github.com/corazawaf/libinjection-go v0.1.2/go.mod h1:OP4TM7xdJ2skyXqNX1AN1wN5nNZEmJNuWbNPOItn7aw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= @@ -170,8 +174,8 @@ github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= -github.com/crowdsecurity/go-cs-lib v0.0.2 h1:+Tjmf/IclOXNzU9sxKVQvUl9CkMfbM60xQ0zA05NWps= -github.com/crowdsecurity/go-cs-lib v0.0.2/go.mod h1:iznTJ19qLTYdZBcRb5RVDlcUdSlayBCivBkWsXlOY3g= +github.com/crowdsecurity/go-cs-lib v0.0.0-20230531105801-4c1535c2b3bd h1:Y70ceDKAKYFXTnxEjXuBDSh07umvDhbX3PCCYhdtsZ0= +github.com/crowdsecurity/go-cs-lib v0.0.0-20230531105801-4c1535c2b3bd/go.mod h1:9JJLSpGj1ZXnROV3xAcJvS/HTaUvuA8K3gGOpO4tfVc= github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4= github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM= github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc= @@ -209,12 +213,12 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -657,18 +661,16 @@ github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= @@ -681,6 +683,7 @@ github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyex github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= @@ -755,6 +758,8 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8= +github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -831,12 +836,8 @@ github.com/segmentio/kafka-go v0.4.34 h1:Dm6YlLMiVSiwwav20KY0AoY63s661FXevwJ3CVH github.com/segmentio/kafka-go v0.4.34/go.mod h1:GAjxBQJdQMB5zfNA21AhpaqOB2Mu+w3De4ni3Gbm8y0= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil/v3 v3.23.5 h1:5SgDCeQ0KW0S4N0znjeM/eFHXXOKyv2dVNgRq/c9P6Y= -github.com/shirou/gopsutil/v3 v3.23.5/go.mod h1:Ng3Maa27Q2KARVJ0SPZF5NdrQSC3XHKP8IIWrHgMeLY= -github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= -github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= -github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= -github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/shirou/gopsutil/v3 v3.22.12 h1:oG0ns6poeUSxf78JtOsfygNWuEHYYz8hnnNg7P04TJs= +github.com/shirou/gopsutil/v3 v3.22.12/go.mod h1:Xd7P1kwZcp5VW52+9XsirIKd/BROzbb2wdX3Kqlz9uI= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -887,6 +888,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -895,13 +897,14 @@ github.com/tetratelabs/wazero v1.0.0-rc.2/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+Gk github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c h1:HelZ2kAFadG0La9d+4htN4HzQ68Bm2iM9qKMSMES6xg= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c/go.mod h1:JlzghshsemAMDGZLytTFY8C1JQxQPhnatWqNwUXjggo= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.13.0 h1:3TFY9yxOQShrvmjdM76K+jc66zJeT6D3/VFFYCGQf7M= -github.com/tidwall/gjson v1.13.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= @@ -940,8 +943,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= -github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zclconf/go-cty v1.8.0 h1:s4AvqaeQzJIu3ndv4gVIhplVD0krU+bgrcLSVUnaWuA= github.com/zclconf/go-cty v1.8.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= @@ -960,8 +963,8 @@ go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= -go.mongodb.org/mongo-driver v1.9.4 h1:qXWlnK2WCOWSxJ/Hm3XyYOGKv3ujA2btBsCyuIFvQjc= -go.mongodb.org/mongo-driver v1.9.4/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= +go.mongodb.org/mongo-driver v1.9.0 h1:f3aLGJvQmBl8d9S40IL+jEyBC6hfLPbJjv9t5hEM9ck= +go.mongodb.org/mongo-driver v1.9.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -1028,8 +1031,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= -golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1052,8 +1055,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= -golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1104,8 +1107,8 @@ golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1125,8 +1128,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1198,23 +1201,23 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1224,8 +1227,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1294,7 +1297,6 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1465,6 +1467,7 @@ k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2R k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/mk/__gmsl b/mk/__gmsl deleted file mode 100644 index 6cd4a3090..000000000 --- a/mk/__gmsl +++ /dev/null @@ -1,969 +0,0 @@ -# ---------------------------------------------------------------------------- -# -# GNU Make Standard Library (GMSL) -# -# A library of functions to be used with GNU Make's $(call) that -# provides functionality not available in standard GNU Make. -# -# Copyright (c) 2005-2022 John Graham-Cumming -# -# This file is part of GMSL -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# Neither the name of the John Graham-Cumming nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# -# ---------------------------------------------------------------------------- - -# This is the GNU Make Standard Library version number as a list with -# three items: major, minor, revision - -gmsl_version := 1 2 0 - -__gmsl_name := GNU Make Standard Library - -# Used to output warnings and error from the library, it's possible to -# disable any warnings or errors by overriding these definitions -# manually or by setting GMSL_NO_WARNINGS or GMSL_NO_ERRORS - -ifdef GMSL_NO_WARNINGS -__gmsl_warning := -else -__gmsl_warning = $(if $1,$(warning $(__gmsl_name): $1)) -endif - -ifdef GMSL_NO_ERRORS -__gmsl_error := -else - __gmsl_error = $(if $1,$(error $(__gmsl_name): $1)) -endif - -# If GMSL_TRACE is enabled then calls to the library functions are -# traced to stdout using warning messages with their arguments - -ifdef GMSL_TRACE -__gmsl_tr1 = $(warning $0('$1')) -__gmsl_tr2 = $(warning $0('$1','$2')) -__gmsl_tr3 = $(warning $0('$1','$2','$3')) -else -__gmsl_tr1 := -__gmsl_tr2 := -__gmsl_tr3 := -endif - -# See if spaces are valid in variable names (this was the case until -# GNU Make 3.82) -ifeq ($(MAKE_VERSION),3.82) -__gmsl_spaced_vars := $(false) -else -__gmsl_spaced_vars := $(true) -endif - -# Figure out whether we have $(eval) or not (GNU Make 3.80 and above) -# if we do not then output a warning message, if we do then some -# functions will be enabled. - -__gmsl_have_eval := $(false) -__gmsl_ignore := $(eval __gmsl_have_eval := $(true)) - -# If this is being run with Electric Cloud's emake then warn that -# their $(eval) support is incomplete in 1.x, 2.x, 3.x, 4.x and 5.0, -# 5.1, 5.2 and 5.3 - -ifdef ECLOUD_BUILD_ID -__gmsl_emake_major := $(word 1,$(subst ., ,$(EMAKE_VERSION))) -__gmsl_emake_minor := $(word 2,$(subst ., ,$(EMAKE_VERSION))) -ifneq ("$(findstring $(__gmsl_emake_major),1 2 3 4)$(findstring $(__gmsl_emake_major)$(__gmsl_emake_minor),50 51 52 53)","") -$(warning You are using a version of Electric Cloud's emake which has incomplete $$(eval) support) -__gmsl_have_eval := $(false) -endif -endif - -# See if we have $(lastword) (GNU Make 3.81 and above) - -__gmsl_have_lastword := $(lastword $(false) $(true)) - -# See if we have native or and and (GNU Make 3.81 and above) - -__or_tt := /$(or $(true),$(true))/$(or $(true),$(false))/$(or $(false),$(true))/$(or $(false),$(false))/ -__and_tt := /$(and $(true),$(true))/$(and $(true),$(false))/$(and $(false),$(true))/$(and $(false),$(false))/ -__gmsl_have_or := $(if $(filter /T/T/T//,$(__or_tt)),$(true),$(false)) -__gmsl_have_and := $(if $(filter /T////,$(__and_tt)),$(true),$(false)) - -ifneq ($(__gmsl_have_eval),$(true)) -$(call __gmsl_warning,Your make version $(MAKE_VERSION) does not support $$$$(eval): some functions disabled) -endif - -__gmsl_dollar := $$ -__gmsl_hash := \# - -# ---------------------------------------------------------------------------- -# ---------------------------------------------------------------------------- -# Function: gmsl_compatible -# Arguments: List containing the desired library version number (maj min rev) -# Returns: $(true) if this version of the library is compatible -# with the requested version number, otherwise $(false) -# ---------------------------------------------------------------------------- -gmsl_compatible = $(strip \ - $(if $(call gt,$(word 1,$1),$(word 1,$(gmsl_version))), \ - $(false), \ - $(if $(call lt,$(word 1,$1),$(word 1,$(gmsl_version))), \ - $(true), \ - $(if $(call gt,$(word 2,$1),$(word 2,$(gmsl_version))), \ - $(false), \ - $(if $(call lt,$(word 2,$1),$(word 2,$(gmsl_version))), \ - $(true), \ - $(call lte,$(word 3,$1),$(word 3,$(gmsl_version)))))))) - -# ########################################################################### -# LOGICAL OPERATORS -# ########################################################################### - -# not is defined in gmsl - -# ---------------------------------------------------------------------------- -# Function: and -# Arguments: Two boolean values -# Returns: Returns $(true) if both of the booleans are true -# ---------------------------------------------------------------------------- -ifneq ($(__gmsl_have_and),$(true)) -and = $(__gmsl_tr2)$(if $1,$(if $2,$(true),$(false)),$(false)) -endif - -# ---------------------------------------------------------------------------- -# Function: or -# Arguments: Two boolean values -# Returns: Returns $(true) if either of the booleans is true -# ---------------------------------------------------------------------------- -ifneq ($(__gmsl_have_or),$(true)) -or = $(__gmsl_tr2)$(if $1$2,$(true),$(false)) -endif - -# ---------------------------------------------------------------------------- -# Function: xor -# Arguments: Two boolean values -# Returns: Returns $(true) if exactly one of the booleans is true -# ---------------------------------------------------------------------------- -xor = $(__gmsl_tr2)$(if $1,$(if $2,$(false),$(true)),$(if $2,$(true),$(false))) - -# ---------------------------------------------------------------------------- -# Function: nand -# Arguments: Two boolean values -# Returns: Returns value of 'not and' -# ---------------------------------------------------------------------------- -nand = $(__gmsl_tr2)$(if $1,$(if $2,$(false),$(true)),$(true)) - -# ---------------------------------------------------------------------------- -# Function: nor -# Arguments: Two boolean values -# Returns: Returns value of 'not or' -# ---------------------------------------------------------------------------- -nor = $(__gmsl_tr2)$(if $1$2,$(false),$(true)) - -# ---------------------------------------------------------------------------- -# Function: xnor -# Arguments: Two boolean values -# Returns: Returns value of 'not xor' -# ---------------------------------------------------------------------------- -xnor =$(__gmsl_tr2)$(if $1,$(if $2,$(true),$(false)),$(if $2,$(false),$(true))) - -# ########################################################################### -# LIST MANIPULATION FUNCTIONS -# ########################################################################### - -# ---------------------------------------------------------------------------- -# Function: first (same as LISP's car, or head) -# Arguments: 1: A list -# Returns: Returns the first element of a list -# ---------------------------------------------------------------------------- -first = $(__gmsl_tr1)$(firstword $1) - -# ---------------------------------------------------------------------------- -# Function: last -# Arguments: 1: A list -# Returns: Returns the last element of a list -# ---------------------------------------------------------------------------- -ifeq ($(__gmsl_have_lastword),$(true)) -last = $(__gmsl_tr1)$(lastword $1) -else -last = $(__gmsl_tr1)$(if $1,$(word $(words $1),$1)) -endif - -# ---------------------------------------------------------------------------- -# Function: rest (same as LISP's cdr, or tail) -# Arguments: 1: A list -# Returns: Returns the list with the first element removed -# ---------------------------------------------------------------------------- -rest = $(__gmsl_tr1)$(wordlist 2,$(words $1),$1) - -# ---------------------------------------------------------------------------- -# Function: chop -# Arguments: 1: A list -# Returns: Returns the list with the last element removed -# ---------------------------------------------------------------------------- -chop = $(__gmsl_tr1)$(wordlist 2,$(words $1),x $1) - -# ---------------------------------------------------------------------------- -# Function: map -# Arguments: 1: Name of function to $(call) for each element of list -# 2: List to iterate over calling the function in 1 -# Returns: The list after calling the function on each element -# ---------------------------------------------------------------------------- -map = $(__gmsl_tr2)$(strip $(foreach a,$2,$(call $1,$a))) - -# ---------------------------------------------------------------------------- -# Function: pairmap -# Arguments: 1: Name of function to $(call) for each pair of elements -# 2: List to iterate over calling the function in 1 -# 3: Second list to iterate over calling the function in 1 -# Returns: The list after calling the function on each pair of elements -# ---------------------------------------------------------------------------- -pairmap = $(strip $(__gmsl_tr3)\ - $(if $2$3,$(call $1,$(call first,$2),$(call first,$3)) \ - $(call pairmap,$1,$(call rest,$2),$(call rest,$3)))) - -# ---------------------------------------------------------------------------- -# Function: leq -# Arguments: 1: A list to compare against... -# 2: ...this list -# Returns: Returns $(true) if the two lists are identical -# ---------------------------------------------------------------------------- -leq = $(__gmsl_tr2)$(strip $(if $(call seq,$(words $1),$(words $2)), \ - $(call __gmsl_list_equal,$1,$2),$(false))) - -__gmsl_list_equal = $(if $(strip $1), \ - $(if $(call seq,$(call first,$1),$(call first,$2)), \ - $(call __gmsl_list_equal, \ - $(call rest,$1), \ - $(call rest,$2)), \ - $(false)), \ - $(true)) - -# ---------------------------------------------------------------------------- -# Function: lne -# Arguments: 1: A list to compare against... -# 2: ...this list -# Returns: Returns $(true) if the two lists are different -# ---------------------------------------------------------------------------- -lne = $(__gmsl_tr2)$(call not,$(call leq,$1,$2)) - -# ---------------------------------------------------------------------------- -# Function: reverse -# Arguments: 1: A list to reverse -# Returns: The list with its elements in reverse order -# ---------------------------------------------------------------------------- -reverse =$(__gmsl_tr1)$(strip $(if $1,$(call reverse,$(call rest,$1)) \ - $(call first,$1))) - -# ---------------------------------------------------------------------------- -# Function: uniq -# Arguments: 1: A list from which to remove repeated elements -# Returns: The list with duplicate elements removed without reordering -# ---------------------------------------------------------------------------- -uniq = $(strip $(__gmsl_tr1) $(if $1,$(firstword $1) \ - $(call uniq,$(filter-out $(firstword $1),$1)))) - -# ---------------------------------------------------------------------------- -# Function: length -# Arguments: 1: A list -# Returns: The number of elements in the list -# ---------------------------------------------------------------------------- -length = $(__gmsl_tr1)$(words $1) - -# ########################################################################### -# STRING MANIPULATION FUNCTIONS -# ########################################################################### - -# Helper function that translates any GNU Make 'true' value (i.e. a -# non-empty string) to our $(true) - -__gmsl_make_bool = $(if $(strip $1),$(true),$(false)) - -# ---------------------------------------------------------------------------- -# Function: seq -# Arguments: 1: A string to compare against... -# 2: ...this string -# Returns: Returns $(true) if the two strings are identical -# ---------------------------------------------------------------------------- -seq = $(__gmsl_tr2)$(if $(subst x$1,,x$2)$(subst x$2,,x$1),$(false),$(true)) - -# ---------------------------------------------------------------------------- -# Function: sne -# Arguments: 1: A string to compare against... -# 2: ...this string -# Returns: Returns $(true) if the two strings are not the same -# ---------------------------------------------------------------------------- -sne = $(__gmsl_tr2)$(call not,$(call seq,$1,$2)) - -# ---------------------------------------------------------------------------- -# Function: split -# Arguments: 1: The character to split on -# 2: A string to split -# Returns: Splits a string into a list separated by spaces at the split -# character in the first argument -# ---------------------------------------------------------------------------- -split = $(__gmsl_tr2)$(strip $(subst $1, ,$2)) - -# ---------------------------------------------------------------------------- -# Function: merge -# Arguments: 1: The character to put between fields -# 2: A list to merge into a string -# Returns: Merges a list into a single string, list elements are separated -# by the character in the first argument -# ---------------------------------------------------------------------------- -merge = $(__gmsl_tr2)$(strip $(if $2, \ - $(if $(call seq,1,$(words $2)), \ - $2,$(call first,$2)$1$(call merge,$1,$(call rest,$2))))) - -ifdef __gmsl_have_eval -# ---------------------------------------------------------------------------- -# Function: tr -# Arguments: 1: The list of characters to translate from -# 2: The list of characters to translate to -# 3: The text to translate -# Returns: Returns the text after translating characters -# ---------------------------------------------------------------------------- -tr = $(strip $(__gmsl_tr3)$(call assert_no_dollar,$0,$1$2$3) \ - $(eval __gmsl_t := $3) \ - $(foreach c, \ - $(join $(addsuffix :,$1),$2), \ - $(eval __gmsl_t := \ - $(subst $(word 1,$(subst :, ,$c)),$(word 2,$(subst :, ,$c)), \ - $(__gmsl_t))))$(__gmsl_t)) - -# Common character classes for use with the tr function. Each of -# these is actually a variable declaration and must be wrapped with -# $() or ${} to be used. - -[A-Z] := A B C D E F G H I J K L M N O P Q R S T U V W X Y Z # -[a-z] := a b c d e f g h i j k l m n o p q r s t u v w x y z # -[0-9] := 0 1 2 3 4 5 6 7 8 9 # -[A-F] := A B C D E F # - -# ---------------------------------------------------------------------------- -# Function: uc -# Arguments: 1: Text to upper case -# Returns: Returns the text in upper case -# ---------------------------------------------------------------------------- -uc = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(call tr,$([a-z]),$([A-Z]),$1) - -# ---------------------------------------------------------------------------- -# Function: lc -# Arguments: 1: Text to lower case -# Returns: Returns the text in lower case -# ---------------------------------------------------------------------------- -lc = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(call tr,$([A-Z]),$([a-z]),$1) - -# ---------------------------------------------------------------------------- -# Function: strlen -# Arguments: 1: A string -# Returns: Returns the length of the string -# ---------------------------------------------------------------------------- - -# This results in __gmsl_tab containing a tab - -__gmsl_tab := # - -__gmsl_characters := A B C D E F G H I J K L M N O P Q R S T U V W X Y Z -__gmsl_characters += a b c d e f g h i j k l m n o p q r s t u v w x y z -__gmsl_characters += 0 1 2 3 4 5 6 7 8 9 -__gmsl_characters += ` ~ ! @ \# $$ % ^ & * ( ) - _ = + -__gmsl_characters += { } [ ] \ : ; ' " < > , . / ? | - -# This results in __gmsl_space containing just a space - -__gmsl_empty := -__gmsl_space := $(__gmsl_empty) $(__gmsl_empty) - -strlen = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(strip $(eval __temp := $(subst $(__gmsl_space),x,$1))$(foreach a,$(__gmsl_characters),$(eval __temp := $$(subst $$a,x,$(__temp))))$(eval __temp := $(subst x,x ,$(__temp)))$(words $(__temp))) - -# This results in __gmsl_newline containing just a newline - -define __gmsl_newline - - -endef - -# ---------------------------------------------------------------------------- -# Function: substr -# Arguments: 1: A string -# 2: Start position (first character is 1) -# 3: End position (inclusive) -# Returns: A substring. -# Note: The string in $1 must not contain a § -# ---------------------------------------------------------------------------- - -substr = $(if $2,$(__gmsl_tr3)$(call assert_no_dollar,$0,$1$2$3)$(strip $(eval __temp := $$(subst $$(__gmsl_space),§ ,$$1))$(foreach a,$(__gmsl_characters),$(eval __temp := $$(subst $$a,$$a$$(__gmsl_space),$(__temp))))$(eval __temp := $(wordlist $2,$3,$(__temp))))$(subst §,$(__gmsl_space),$(subst $(__gmsl_space),,$(__temp)))) - -endif # __gmsl_have_eval - -# ########################################################################### -# SET MANIPULATION FUNCTIONS -# ########################################################################### - -# Sets are represented by sorted, deduplicated lists. To create a set -# from a list use set_create, or start with the empty_set and -# set_insert individual elements - -# This is the empty set -empty_set := - -# ---------------------------------------------------------------------------- -# Function: set_create -# Arguments: 1: A list of set elements -# Returns: Returns the newly created set -# ---------------------------------------------------------------------------- -set_create = $(__gmsl_tr1)$(sort $1) - -# ---------------------------------------------------------------------------- -# Function: set_insert -# Arguments: 1: A single element to add to a set -# 2: A set -# Returns: Returns the set with the element added -# ---------------------------------------------------------------------------- -set_insert = $(__gmsl_tr2)$(sort $1 $2) - -# ---------------------------------------------------------------------------- -# Function: set_remove -# Arguments: 1: A single element to remove from a set -# 2: A set -# Returns: Returns the set with the element removed -# ---------------------------------------------------------------------------- -set_remove = $(__gmsl_tr2)$(filter-out $1,$2) - -# ---------------------------------------------------------------------------- -# Function: set_is_member, set_is_not_member -# Arguments: 1: A single element -# 2: A set -# Returns: (set_is_member) Returns $(true) if the element is in the set -# (set_is_not_member) Returns $(false) if the element is in the set -# ---------------------------------------------------------------------------- -set_is_member = $(__gmsl_tr2)$(if $(filter $1,$2),$(true),$(false)) -set_is_not_member = $(__gmsl_tr2)$(if $(filter $1,$2),$(false),$(true)) - -# ---------------------------------------------------------------------------- -# Function: set_union -# Arguments: 1: A set -# 2: Another set -# Returns: Returns the union of the two sets -# ---------------------------------------------------------------------------- -set_union = $(__gmsl_tr2)$(sort $1 $2) - -# ---------------------------------------------------------------------------- -# Function: set_intersection -# Arguments: 1: A set -# 2: Another set -# Returns: Returns the intersection of the two sets -# ---------------------------------------------------------------------------- -set_intersection = $(__gmsl_tr2)$(filter $1,$2) - -# ---------------------------------------------------------------------------- -# Function: set_is_subset -# Arguments: 1: A set -# 2: Another set -# Returns: Returns $(true) if the first set is a subset of the second -# ---------------------------------------------------------------------------- -set_is_subset = $(__gmsl_tr2)$(call set_equal,$(call set_intersection,$1,$2),$1) - -# ---------------------------------------------------------------------------- -# Function: set_equal -# Arguments: 1: A set -# 2: Another set -# Returns: Returns $(true) if the two sets are identical -# ---------------------------------------------------------------------------- -set_equal = $(__gmsl_tr2)$(call seq,$1,$2) - -# ########################################################################### -# ARITHMETIC LIBRARY -# ########################################################################### - -# Integers a represented by lists with the equivalent number of x's. -# For example the number 4 is x x x x. - -# ---------------------------------------------------------------------------- -# Function: int_decode -# Arguments: 1: A number of x's representation -# Returns: Returns the integer for human consumption that is represented -# by the string of x's -# ---------------------------------------------------------------------------- -int_decode = $(__gmsl_tr1)$(if $1,$(if $(call seq,$(word 1,$1),x),$(words $1),$1),0) - -# ---------------------------------------------------------------------------- -# Function: int_encode -# Arguments: 1: A number in human-readable integer form -# Returns: Returns the integer encoded as a string of x's -# ---------------------------------------------------------------------------- -__int_encode = $(if $1,$(if $(call seq,$(words $(wordlist 1,$1,$2)),$1),$(wordlist 1,$1,$2),$(call __int_encode,$1,$(if $2,$2 $2,x)))) -__strip_leading_zero = $(if $1,$(if $(call seq,$(patsubst 0%,%,$1),$1),$1,$(call __strip_leading_zero,$(patsubst 0%,%,$1))),0) -int_encode = $(__gmsl_tr1)$(call __int_encode,$(call __strip_leading_zero,$1)) - -# The arithmetic library functions come in two forms: one form of each -# function takes integers as arguments and the other form takes the -# encoded form (x's created by a call to int_encode). For example, -# there are two plus functions: -# -# plus Called with integer arguments and returns an integer -# int_plus Called with encoded arguments and returns an encoded result -# -# plus will be slower than int_plus because its arguments and result -# have to be translated between the x's format and integers. If doing -# a complex calculation use the int_* forms with a single encoding of -# inputs and single decoding of the output. For simple calculations -# the direct forms can be used. - -# Helper function used to wrap an int_* function into a function that -# takes a pair of integers, perhaps a function and returns an integer -# result -__gmsl_int_wrap = $(call int_decode,$(call $1,$(call int_encode,$2),$(call int_encode,$3))) -__gmsl_int_wrap1 = $(call int_decode,$(call $1,$(call int_encode,$2))) -__gmsl_int_wrap2 = $(call $1,$(call int_encode,$2),$(call int_encode,$3)) - -# ---------------------------------------------------------------------------- -# Function: int_plus -# Arguments: 1: A number in x's representation -# 2: Another number in x's represntation -# Returns: Returns the sum of the two numbers in x's representation -# ---------------------------------------------------------------------------- -int_plus = $(strip $(__gmsl_tr2)$1 $2) - -# ---------------------------------------------------------------------------- -# Function: plus (wrapped version of int_plus) -# Arguments: 1: An integer -# 2: Another integer -# Returns: Returns the sum of the two integers -# ---------------------------------------------------------------------------- -plus = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_plus,$1,$2) - -# ---------------------------------------------------------------------------- -# Function: int_subtract -# Arguments: 1: A number in x's representation -# 2: Another number in x's represntation -# Returns: Returns the difference of the two numbers in x's representation, -# or outputs an error on a numeric underflow -# ---------------------------------------------------------------------------- -int_subtract = $(strip $(__gmsl_tr2)$(if $(call int_gte,$1,$2), \ - $(filter-out xx,$(join $1,$2)), \ - $(call __gmsl_warning,Subtraction underflow))) - -# ---------------------------------------------------------------------------- -# Function: subtract (wrapped version of int_subtract) -# Arguments: 1: An integer -# 2: Another integer -# Returns: Returns the difference of the two integers, -# or outputs an error on a numeric underflow -# ---------------------------------------------------------------------------- -subtract = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_subtract,$1,$2) - -# ---------------------------------------------------------------------------- -# Function: int_multiply -# Arguments: 1: A number in x's representation -# 2: Another number in x's represntation -# Returns: Returns the product of the two numbers in x's representation -# ---------------------------------------------------------------------------- -int_multiply = $(strip $(__gmsl_tr2)$(foreach a,$1,$2)) - -# ---------------------------------------------------------------------------- -# Function: multiply (wrapped version of int_multiply) -# Arguments: 1: An integer -# 2: Another integer -# Returns: Returns the product of the two integers -# ---------------------------------------------------------------------------- -multiply = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_multiply,$1,$2) - -# ---------------------------------------------------------------------------- -# Function: int_divide -# Arguments: 1: A number in x's representation -# 2: Another number in x's represntation -# Returns: Returns the result of integer division of argument 1 divided -# by argument 2 in x's representation -# ---------------------------------------------------------------------------- -int_divide = $(__gmsl_tr2)$(strip $(if $1,$(if $2, \ - $(subst M,x,$(filter-out x,$(subst $2,M,$1))), \ - $(call __gmsl_error,Division by zero)))) - -# ---------------------------------------------------------------------------- -# Function: divide (wrapped version of int_divide) -# Arguments: 1: An integer -# 2: Another integer -# Returns: Returns the integer division of the first argument by the second -# ---------------------------------------------------------------------------- -divide = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_divide,$1,$2) - -# ---------------------------------------------------------------------------- -# Function: int_modulo -# Arguments: 1: A number in x's representation -# 2: Another number in x's represntation -# Returns: Returns the remainder of integer division of argument 1 divided -# by argument 2 in x's representation -# ---------------------------------------------------------------------------- -int_modulo = $(__gmsl_tr2)$(strip $(if $1,$(if $2, \ - $(filter-out M,$(subst $2,M,$1)), \ - $(call __gmsl_error,Division by zero)))) - -# ---------------------------------------------------------------------------- -# Function: modulo (wrapped version of int_modulo) -# Arguments: 1: An integer -# 2: Another integer -# Returns: Returns the remainder of integer division of the first argument -# by the second -# ---------------------------------------------------------------------------- -modulo = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_modulo,$1,$2) - -# ---------------------------------------------------------------------------- -# Function: int_max, int_min -# Arguments: 1: A number in x's representation -# 2: Another number in x's represntation -# Returns: Returns the maximum or minimum of its arguments in x's -# representation -# ---------------------------------------------------------------------------- -int_max = $(__gmsl_tr2)$(subst xx,x,$(join $1,$2)) -int_min = $(__gmsl_tr2)$(subst xx,x,$(filter xx,$(join $1,$2))) - -# ---------------------------------------------------------------------------- -# Function: max, min -# Arguments: 1: An integer -# 2: Another integer -# Returns: Returns the maximum or minimum of its integer arguments -# ---------------------------------------------------------------------------- -max = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_max,$1,$2) -min = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_min,$1,$2) - -# ---------------------------------------------------------------------------- -# Function: int_gt, int_gte, int_lt, int_lte, int_eq, int_ne -# Arguments: Two x's representation numbers to be compared -# Returns: $(true) or $(false) -# -# int_gt First argument greater than second argument -# int_gte First argument greater than or equal to second argument -# int_lt First argument less than second argument -# int_lte First argument less than or equal to second argument -# int_eq First argument is numerically equal to the second argument -# int_ne First argument is not numerically equal to the second argument -# ---------------------------------------------------------------------------- -int_gt = $(__gmsl_tr2)$(call __gmsl_make_bool, \ - $(filter-out $(words $2), \ - $(words $(call int_max,$1,$2)))) -int_gte = $(__gmsl_tr2)$(call __gmsl_make_bool, \ - $(call int_gt,$1,$2)$(call int_eq,$1,$2)) -int_lt = $(__gmsl_tr2)$(call __gmsl_make_bool, \ - $(filter-out $(words $1), \ - $(words $(call int_max,$1,$2)))) -int_lte = $(__gmsl_tr2)$(call __gmsl_make_bool, \ - $(call int_lt,$1,$2)$(call int_eq,$1,$2)) -int_eq = $(__gmsl_tr2)$(call __gmsl_make_bool, \ - $(filter $(words $1),$(words $2))) -int_ne = $(__gmsl_tr2)$(call __gmsl_make_bool, \ - $(filter-out $(words $1),$(words $2))) - -# ---------------------------------------------------------------------------- -# Function: gt, gte, lt, lte, eq, ne -# Arguments: Two integers to be compared -# Returns: $(true) or $(false) -# -# gt First argument greater than second argument -# gte First argument greater than or equal to second argument -# lt First argument less than second argument -# lte First argument less than or equal to second argument -# eq First argument is numerically equal to the second argument -# ne First argument is not numerically equal to the second argument -# ---------------------------------------------------------------------------- -gt = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_gt,$1,$2) -gte = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_gte,$1,$2) -lt = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_lt,$1,$2) -lte = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_lte,$1,$2) -eq = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_eq,$1,$2) -ne = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_ne,$1,$2) - -# increment adds 1 to its argument, decrement subtracts 1. Note that -# decrement does not range check and hence will not underflow, but -# will incorrectly say that 0 - 1 = 0 - -# ---------------------------------------------------------------------------- -# Function: int_inc -# Arguments: 1: A number in x's representation -# Returns: The number incremented by 1 in x's representation -# ---------------------------------------------------------------------------- -int_inc = $(strip $(__gmsl_tr1)$1 x) - -# ---------------------------------------------------------------------------- -# Function: inc -# Arguments: 1: An integer -# Returns: The argument incremented by 1 -# ---------------------------------------------------------------------------- -inc = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_inc,$1) - -# ---------------------------------------------------------------------------- -# Function: int_dec -# Arguments: 1: A number in x's representation -# Returns: The number decremented by 1 in x's representation -# ---------------------------------------------------------------------------- -int_dec = $(__gmsl_tr1)$(strip \ - $(if $(call sne,0,$(words $1)), \ - $(wordlist 2,$(words $1),$1))) - -# ---------------------------------------------------------------------------- -# Function: dec -# Arguments: 1: An integer -# Returns: The argument decremented by 1 -# ---------------------------------------------------------------------------- -dec = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_dec,$1) - -# double doubles its argument, and halve halves it - -# ---------------------------------------------------------------------------- -# Function: int_double -# Arguments: 1: A number in x's representation -# Returns: The number doubled (i.e. * 2) and returned in x's representation -# ---------------------------------------------------------------------------- -int_double = $(strip $(__gmsl_tr1)$1 $1) - -# ---------------------------------------------------------------------------- -# Function: double -# Arguments: 1: An integer -# Returns: The integer times 2 -# ---------------------------------------------------------------------------- -double = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_double,$1) - -# ---------------------------------------------------------------------------- -# Function: int_halve -# Arguments: 1: A number in x's representation -# Returns: The number halved (i.e. / 2) and returned in x's representation -# ---------------------------------------------------------------------------- -int_halve = $(__gmsl_tr1)$(strip $(subst xx,x,$(filter-out xy x y, \ - $(join $1,$(foreach a,$1,y x))))) - -# ---------------------------------------------------------------------------- -# Function: halve -# Arguments: 1: An integer -# Returns: The integer divided by 2 -# ---------------------------------------------------------------------------- -halve = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_halve,$1) - -# ---------------------------------------------------------------------------- -# Function: sequence -# Arguments: 1: An integer -# 2: An integer -# Returns: The sequence [arg1, arg2] of integers if arg1 < arg2 or -# [arg2, arg1] if arg2 > arg1. If arg1 == arg1 return [arg1] -# ---------------------------------------------------------------------------- -sequence = $(__gmsl_tr2)$(strip $(if $(call lte,$1,$2), \ - $(call __gmsl_sequence_up,$1,$2), \ - $(call __gmsl_sequence_dn,$2,$1))) - -__gmsl_sequence_up = $(if $(call seq,$1,$2),$1,$1 $(call __gmsl_sequence_up,$(call inc,$1),$2)) -__gmsl_sequence_dn = $(if $(call seq,$1,$2),$1,$2 $(call __gmsl_sequence_dn,$1,$(call dec,$2))) - -# ---------------------------------------------------------------------------- -# Function: dec2hex, dec2bin, dec2oct -# Arguments: 1: An integer -# Returns: The decimal argument converted to hexadecimal, binary or -# octal -# ---------------------------------------------------------------------------- - -__gmsl_digit = $(subst 15,f,$(subst 14,e,$(subst 13,d,$(subst 12,c,$(subst 11,b,$(subst 10,a,$1)))))) - -dec2hex = $(call __gmsl_dec2base,$(call int_encode,$1),$(call int_encode,16)) -dec2bin = $(call __gmsl_dec2base,$(call int_encode,$1),$(call int_encode,2)) -dec2oct = $(call __gmsl_dec2base,$(call int_encode,$1),$(call int_encode,8)) - -__gmsl_base_divide = $(subst $2,X ,$1) -__gmsl_q = $(strip $(filter X,$1)) -__gmsl_r = $(words $(filter x,$1)) - -__gmsl_dec2base = $(eval __gmsl_temp := $(call __gmsl_base_divide,$1,$2))$(call __gmsl_dec2base_,$(call __gmsl_q,$(__gmsl_temp)),$(call __gmsl_r,$(__gmsl_temp)),$2) -__gmsl_dec2base_ = $(if $1,$(call __gmsl_dec2base,$(subst X,x,$1),$3))$(call __gmsl_digit,$2) - -ifdef __gmsl_have_eval -# ########################################################################### -# ASSOCIATIVE ARRAYS -# ########################################################################### - -# Magic string that is very unlikely to appear in a key or value - -__gmsl_aa_magic := faf192c8efbc25c27992c5bc5add390393d583c6 - -# ---------------------------------------------------------------------------- -# Function: set -# Arguments: 1: Name of associative array -# 2: The key value to associate -# 3: The value associated with the key -# Returns: Nothing -# ---------------------------------------------------------------------------- -set = $(__gmsl_tr3)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2$3)$(eval __gmsl_aa_$1_$(__gmsl_aa_magic)_$2_gmsl_aa_$1 := $3) - -# Only used internally by memoize function - -__gmsl_set = $(call set,$1,$2,$3)$3 - -# ---------------------------------------------------------------------------- -# Function: get -# Arguments: 1: Name of associative array -# 2: The key to retrieve -# Returns: The value stored in the array for that key -# ---------------------------------------------------------------------------- -get = $(strip $(__gmsl_tr2)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2)$(__gmsl_aa_$1_$(__gmsl_aa_magic)_$2_gmsl_aa_$1)) - -# ---------------------------------------------------------------------------- -# Function: keys -# Arguments: 1: Name of associative array -# Returns: Returns a list of all defined keys in the array -# ---------------------------------------------------------------------------- -keys = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(sort $(patsubst __gmsl_aa_$1_$(__gmsl_aa_magic)_%_gmsl_aa_$1,%, \ - $(filter __gmsl_aa_$1_$(__gmsl_aa_magic)_%_gmsl_aa_$1,$(.VARIABLES)))) - -# ---------------------------------------------------------------------------- -# Function: defined -# Arguments: 1: Name of associative array -# 2: The key to test -# Returns: Returns true if the key is defined (i.e. not empty) -# ---------------------------------------------------------------------------- -defined = $(__gmsl_tr2)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2)$(call sne,$(call get,$1,$2),) - -endif # __gmsl_have_eval - -ifdef __gmsl_have_eval -# ########################################################################### -# NAMED STACKS -# ########################################################################### - -# ---------------------------------------------------------------------------- -# Function: push -# Arguments: 1: Name of stack -# 2: Value to push onto the top of the stack (must not contain -# a space) -# Returns: None -# ---------------------------------------------------------------------------- -push = $(__gmsl_tr2)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2)$(eval __gmsl_stack_$1 := $2 $(if $(filter-out undefined,\ - $(origin __gmsl_stack_$1)),$(__gmsl_stack_$1))) - -# ---------------------------------------------------------------------------- -# Function: pop -# Arguments: 1: Name of stack -# Returns: Top element from the stack after removing it -# ---------------------------------------------------------------------------- -pop = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(strip $(if $(filter-out undefined,$(origin __gmsl_stack_$1)), \ - $(call first,$(__gmsl_stack_$1)) \ - $(eval __gmsl_stack_$1 := $(call rest,$(__gmsl_stack_$1))))) - -# ---------------------------------------------------------------------------- -# Function: peek -# Arguments: 1: Name of stack -# Returns: Top element from the stack without removing it -# ---------------------------------------------------------------------------- -peek = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(call first,$(__gmsl_stack_$1)) - -# ---------------------------------------------------------------------------- -# Function: depth -# Arguments: 1: Name of stack -# Returns: Number of items on the stack -# ---------------------------------------------------------------------------- -depth = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(words $(__gmsl_stack_$1)) - -endif # __gmsl_have_eval - -ifdef __gmsl_have_eval -# ########################################################################### -# STRING CACHE -# ########################################################################### - -# ---------------------------------------------------------------------------- -# Function: memoize -# Arguments: 1. Name of the function to be called if the string -# has not been previously seen -# 2. A string -# Returns: Returns the result of a memo function (which the user must -# define) on the passed in string and remembers the result. -# -# Example: Set memo = $(shell echo "$1" | md5sum) to make a cache -# of MD5 hashes of strings. $(call memoize,memo,foo bar baz) -# ---------------------------------------------------------------------------- -__gmsl_memoize = $(subst $(__gmsl_space),§,$1)cc2af1bb7c4482f2ba75e338b963d3e7$(subst $(__gmsl_space),§,$2) -memoize = $(__gmsl_tr2)$(strip $(if $(call defined,__gmsl_m,$(__gmsl_memoize)),\ - $(call get,__gmsl_m,$(__gmsl_memoize)), \ - $(call __gmsl_set,__gmsl_m,$(__gmsl_memoize),$(call $1,$2)))) - -endif # __gmsl_have_eval - -# ########################################################################### -# DEBUGGING FACILITIES -# ########################################################################### - -# ---------------------------------------------------------------------------- -# Target: gmsl-echo-% -# Arguments: The % should be replaced by the name of a variable that you -# wish to print out. -# Action: Echos the value of the variable that matches the %. -# For example, 'make gmsl-echo-SHELL' will output the value of -# the SHELL variable. -# ---------------------------------------------------------------------------- -gmsl-echo-%: ; @echo $($*) - -# ---------------------------------------------------------------------------- -# Target: gmsl-print-% -# Arguments: The % should be replaced by the name of a variable that you -# wish to print out. -# Action: Echos the name of the variable that matches the % and its value. -# For example, 'make gmsl-print-SHELL' will output the value of -# the SHELL variable -# ---------------------------------------------------------------------------- -gmsl-print-%: ; @echo $* = $($*) - -# ---------------------------------------------------------------------------- -# Function: assert -# Arguments: 1: A boolean that must be true or the assertion will fail -# 2: The message to print with the assertion -# Returns: None -# ---------------------------------------------------------------------------- -assert = $(if $2,$(if $1,,$(call __gmsl_error,Assertion failure: $2))) - -# ---------------------------------------------------------------------------- -# Function: assert_exists -# Arguments: 1: Name of file that must exist, if it is missing an assertion -# will be generated -# Returns: None -# ---------------------------------------------------------------------------- -assert_exists = $(if $0,$(call assert,$(wildcard $1),file '$1' missing)) - -# ---------------------------------------------------------------------------- -# Function: assert_no_dollar -# Arguments: 1: Name of a function being executd -# 2: Arguments to check -# Returns: None -# ---------------------------------------------------------------------------- -assert_no_dollar = $(call __gmsl_tr2)$(call assert,$(call not,$(findstring $(__gmsl_dollar),$2)),$1 called with a dollar sign in argument) - -# ---------------------------------------------------------------------------- -# Function: assert_no_space -# Arguments: 1: Name of a function being executd -# 2: Arguments to check -# Returns: None -# ---------------------------------------------------------------------------- -ifeq ($(__gmsl_spaced_vars),$(false)) -assert_no_space = $(call assert,$(call not,$(findstring $(__gmsl_aa_magic),$(subst $(__gmsl_space),$(__gmsl_aa_magic),$2))),$1 called with a space in argument) -else -assert_no_space = -endif diff --git a/mk/gmsl b/mk/gmsl deleted file mode 100644 index b22949a31..000000000 --- a/mk/gmsl +++ /dev/null @@ -1,85 +0,0 @@ -# ---------------------------------------------------------------------------- -# -# GNU Make Standard Library (GMSL) -# -# A library of functions to be used with GNU Make's $(call) that -# provides functionality not available in standard GNU Make. -# -# Copyright (c) 2005-2022 John Graham-Cumming -# -# This file is part of GMSL -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# Neither the name of the John Graham-Cumming nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# -# ---------------------------------------------------------------------------- - -# Determine if the library has already been included and if so don't -# bother including it again - -ifndef __gmsl_included - -# Standard definitions for true and false. true is any non-empty -# string, false is an empty string. These are intended for use with -# $(if). - -true := T -false := - -# ---------------------------------------------------------------------------- -# Function: not -# Arguments: 1: A boolean value -# Returns: Returns the opposite of the arg. (true -> false, false -> true) -# ---------------------------------------------------------------------------- -not = $(if $1,$(false),$(true)) - -# Prevent reinclusion of the library - -__gmsl_included := $(true) - -# Try to determine where this file is located. If the caller did -# include /foo/gmsl then extract the /foo/ so that __gmsl gets -# included transparently - -__gmsl_root := - -ifneq ($(MAKEFILE_LIST),) -__gmsl_root := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST)) - -# If there are any spaces in the path in __gmsl_root then give up - -ifeq (1,$(words $(__gmsl_root))) -__gmsl_root := $(patsubst %gmsl,%,$(__gmsl_root)) -endif - -endif - -include $(__gmsl_root)__gmsl - -endif # __gmsl_included - diff --git a/mk/gmsl.html b/mk/gmsl.html deleted file mode 100644 index a8da46987..000000000 --- a/mk/gmsl.html +++ /dev/null @@ -1,733 +0,0 @@ - - - - GNU Make Standard Library - - -

GNU Make Standard Library

-The GNU Make Standard Library (GMSL) is a collection of functions -implemented using native GNU Make functionality that provide list and -string manipulation, integer arithmetic, associative arrays, stacks, -and debugging facilities.  The GMSL is released under the BSD License.
-
-[Project Page] -[Releases] -
-

Using GMSL

-The two files needed are gmsl -and __gmsl.  To -include the GMSL in your Makefile do
-
include gmsl
-gmsl automatically includes __gmsl.  To check that -you have the right version of gmsl -use the gmsl_compatible -function (see -below). The current version is 1 2 0.
-
-The GMSL package also includes a test suite for GMSL.  Just run make -f gmsl-tests.
-

Logical Operators

GMSL has boolean $(true) (a non-empty string) -and $(false) (an empty string).  The following operators can be -used with those variables.
-
-
not
- -
- -Arguments: A boolean value
- -Returns:   Returns $(true) if the boolean is $(false) and vice versa
- -
and
-
-Arguments: Two boolean values
-Returns:   Returns $(true) if both of the booleans are true
-
or
-
-Arguments: Two boolean values
-Returns:   Returns $(true) if either of the booleans is true
-
xor
-
-Arguments: Two boolean values
-Returns:   Returns $(true) if exactly one of the booleans is true
-
nand
-
-Arguments: Two boolean values
-Returns:   Returns value of 'not and'
-
nor
-
-Arguments: Two boolean values
-Returns:   Returns value of 'not or'
-
xnor
-
-Arguments: Two boolean values
-Returns:   Returns value of 'not xor'
-
-

List Manipulation Functions

- A list is a string of characters; the list separator is a space.
- -
-
first
-
-Arguments: 1: A list
-Returns:   Returns the first element of a list
- -
last
-
-Arguments: 1: A list
-Returns:   Returns the last element of a list
- -
rest
-
-Arguments: 1: A list
-Returns:   Returns the list with the first element -removed
- -
chop
-
-Arguments: 1: A list
-Returns:   Returns the list with the last element removed
- -
map
-
-Arguments: 1: Name of function to -$(call) for each element of list
-           2: List to -iterate over calling the function in 1
-Returns:   The list after calling the function on each -element
- -
pairmap
-
-Arguments: 1: Name of function to -$(call) for each pair of elements
-           2: List to -iterate over calling the function in 1
-           3: Second -list to iterate over calling the function in 1
-Returns:   The list after calling the function on each -pair of elements
- -
leq
-
-Arguments: 1: A list to compare -against...
-           2: ...this -list
-Returns:   Returns $(true) if the two lists are identical
- -
lne
-
-Arguments: 1: A list to compare -against...
-           2: ...this -list
-Returns:   Returns $(true) if the two lists are different
- -
reverse
-
-Arguments: 1: A list to reverse
-Returns:   The list with its elements in reverse order
- -
uniq
-
-Arguments: 1: A list to deduplicate
-Returns:   The list with elements in order without duplicates
- -
length
-
-Arguments: 1: A list
-Returns:   The number of elements in the list
- -
-

String Manipulation Functions

-A string is any sequence of characters.
-
-
seq
-
-Arguments: 1: A string to compare -against...
-           2: ...this -string
-Returns:   Returns $(true) if the two strings are -identical
- -
sne
-
-Arguments: 1: A string to compare -against...
-           2: ...this -string
-Returns:   Returns $(true) if the two strings are not -the same
- -
strlen
-
-Arguments: 1: A string
-Returns:   Returns the length of the string
- -
substr
-
-Arguments: 1: A string
-           2: Start offset (first character is 1)
-           3: Ending offset (inclusive)
Returns:   Returns a substring
- -
split
-
-Arguments: 1: The character to -split on
-           2: A -string to split
-Returns:   Splits a string into a list separated by -spaces at the split
-           character -in the first argument
- -
merge
-
-Arguments: 1: The character to -put between fields
-           2: A list -to merge into a string
-Returns:   Merges a list into a single string, list -elements are separated
-           by the -character in the first argument
- -
tr
-
-Arguments: 1: The list of -characters to translate from
-           2: The -list of characters to translate to
-           3: The -text to translate
-Returns:   Returns the text after translating characters
- -
uc
-
-Arguments: 1: Text to upper case
-Returns:   Returns the text in upper case
- -
lc
-
-Arguments: 1: Text to lower case
-Returns:   Returns the text in lower case
- -
-

Set Manipulation Functions

-Sets are represented by sorted, deduplicated lists. To create a set -from a list use set_create, or start with the empty_set and set_insert individual elements. -The empty set is defined as empty_set.

- -

set_create
-
-Arguments: 1: A list of set elements
-Returns:   Returns the newly created set
- - -
set_insert
-
-Arguments: 1: A single element to add to a set
-           2: A set
-Returns:   Returns the set with the element added
- - -
set_remove
-
-Arguments: 1: A single element to remove from a set
-           2: A set
-Returns:   Returns the set with the element removed
- - -
set_is_member
-
-Arguments: 1: A single element
-           2: A set
-Returns:   Returns $(true) if the element is in the set
- - -
set_is_not_member
-
-Arguments: 1: A single element
-           2: A set
-Returns:   Returns $(false) if the element is in the set
- - -
set_union
-
-Arguments: 1: A set
-           2: Another set
-Returns:   Returns the union of the two sets
- - -
set_intersection
-
-Arguments: 1: A set
-           2: Another set
-Returns:   Returns the intersection of the two sets
- - -
set_is_subset
-
-Arguments: 1: A set
-           2: Another set
-Returns:   Returns $(true) if the first set is a subset of the second
- - -
set_equal
-
-Arguments: 1: A set
-           2: Another set
-Returns:   Returns $(true) if the two sets are identical
- - -
-

Integer Arithmetic Functions

-Integers are represented by lists with the equivalent number of -x's.  For example the number 4 is x x x x.  The maximum -integer that the library can handle as input (i.e. as the argument to a -call to int_encode) is -65536. There is no limit on integer size for internal computations or -output.
-
-The arithmetic library functions come in two forms: one form of each -function takes integers as arguments and the other form takes the -encoded form (x's created by a call to int_encode).  For example, -there are two plus functions: plus -(called with integer arguments and returns an integer) and int_plus (called with encoded -arguments and returns an encoded result).
-
-plus will be slower than int_plus because its arguments -and result have to be translated between the x's format and -integers.  If doing a complex calculation use the int_* forms with a single -encoding of inputs and single decoding of the output.  For simple -calculations the direct forms can be used.
-
-
int_decode
-
-Arguments: 1: A number of x's -representation
-Returns:   Returns the integer for human consumption -that is represented
-           by the -string of x's
- -
int_encode
-
-Arguments: 1: A number in -human-readable integer form
-Returns:   Returns the integer encoded as a string of x's
- -
int_plus
-
-Arguments: 1: A number in x's -representation
-           2: Another -number in x's represntation
-Returns:   Returns the sum of the two numbers in x's -representation
- -
plus (wrapped version of int_plus)
-
-Arguments: 1: An integer
-           2: Another -integer
-Returns:   Returns the sum of the two integers
- -
int_subtract
-
-Arguments: 1: A number in x's -representation
-           2: Another -number in x's represntation
-Returns:   Returns the difference of the two numbers in -x's representation,
-           or outputs -an error on a numeric underflow
- -
subtract (wrapped version of int_subtract)
-
-Arguments: 1: An integer
-           2: Another -integer
-Returns:   Returns the difference of the two integers,
-           or outputs -an error on a numeric underflow
- -
int_multiply
-
-Arguments: 1: A number in x's -representation
-           2: Another -number in x's represntation
-Returns:   Returns the product of the two numbers in x's -representation
- -
multiply (wrapped version of int_multiply)
-
-Arguments: 1: An integer
-           2: Another -integer
-Returns:   Returns the product of the two integers
- -
int_divide
-
-Arguments: 1: A number in x's -representation
-           2: Another -number in x's represntation
-Returns:   Returns the result of integer division of -argument 1 divided
-           by -argument 2 in x's representation
- -
divide (wrapped version of int_divide)
-
-Arguments: 1: An integer
-           2: Another -integer
-Returns:   Returns the integer division of the first -argument by the second
- -
int_modulo
-
-Arguments: 1: A number in x's -representation
-           2: Another -number in x's represntation
-Returns:   Returns the remainder of integer division of -argument 1 divided
-           by -argument 2 in x's representation
- -
modulo (wrapped version of int_modulo)
-
-Arguments: 1: An integer
-           2: Another -integer
-Returns:   Returns the remainder of integer division of the first -argument by the second
- -
int_max, int_min
-
-Arguments: 1: A number in x's -representation
-           2: Another -number in x's represntation
-Returns:   Returns the maximum or minimum of its -arguments in x's
-           -representation
- -
max, min
-
-Arguments: 1: An integer
-           2: Another -integer
-Returns:   Returns the maximum or minimum of its integer -arguments
- -
int_gt, int_gte, int_lt, int_lte, int_eq, int_ne
-
-Arguments: Two x's representation -numbers to be compared
-Returns:   $(true) or $(false)
-
-int_gt First argument greater than second argument
-int_gte First argument greater than or equal to second argument
-int_lt First argument less than second argument
-int_lte First argument less than or equal to second argument
-int_eq First argument is numerically equal to the second argument
-int_ne First argument is not numerically equal to the second argument
- -
gt, gte, lt, lte, eq, ne
-
-Arguments: Two integers to be -compared
-Returns:   $(true) or $(false)
-
-gt First argument greater than second argument
-gte First argument greater than or equal to second argument
-lt First argument less than second argument
-lte First argument less than or equal to second argument
-eq First argument is numerically equal to the second argument
-ne First argument is not numerically equal to the second argument
- -increment adds 1 to its argument, decrement subtracts 1. Note that
-decrement does not range check and hence will not underflow, but
-will incorrectly say that 0 - 1 = 0
-
int_inc
-
-Arguments: 1: A number in x's -representation
-Returns:   The number incremented by 1 in x's -representation
- -
inc
-
-Arguments: 1: An integer
-Returns:   The argument incremented by 1
- -
int_dec
-
-Arguments: 1: A number in x's -representation
-Returns:   The number decremented by 1 in x's -representation
- -
dec
-
-Arguments: 1: An integer
-Returns:   The argument decremented by 1
- -
int_double
-
-Arguments: 1: A number in x's -representation
-Returns:   The number doubled (i.e. * 2) and returned in -x's representation
- -
double
-
-Arguments: 1: An integer
-Returns:   The integer times 2
- -
int_halve
-
-Arguments: 1: A number in x's -representation
-Returns:   The number halved (i.e. / 2) and returned in -x's representation
- -
halve
-
-Arguments: 1: An integer
-Returns:   The integer divided by 2
- -
sequence
-
-Arguments: 1: An integer
-           2: An integer
-Returns:   The sequence [arg1 arg2] if arg1 >= arg2 or [arg2 arg1] if arg2 > arg1
- -
dec2hex, dec2bin, dec2oct
-
-Arguments: 1: An integer
-Returns:   The decimal argument converted to hexadecimal, binary or octal
- -
-

Associative Arrays

-An associate array maps a key value (a string with no spaces in it) to -a single value (any string).   
-
- -
set
-
-Arguments: 1: Name of associative -array
-           2: The key -value to associate
-           3: The -value associated with the key
-Returns:   Nothing
- -
get
-
-Arguments: 1: Name of associative -array
-           2: The key -to retrieve
-Returns:   The value stored in the array for that key
- -
keys
-
-Arguments: 1: Name of associative -array
-Returns:   Returns a list of all defined keys in the -array
- -
defined
-
-Arguments: 1: Name of associative -array
-           2: The key -to test
-Returns:   Returns true if the key is defined (i.e. not -empty)
- -
-

Named Stacks

-A stack is an ordered list of strings (with no spaces in them).
-
-
push
-
-Arguments: 1: Name of stack
-           2: Value -to push onto the top of the stack (must not contain
-           a space)
-Returns:   None
- -
pop
-
-Arguments: 1: Name of stack
-Returns:   Top element from the stack after removing it
- -
peek
-
-Arguments: 1: Name of stack
-Returns:   Top element from the stack without removing it
- -
depth
-
-Arguments: 1: Name of stack
-Returns:   Number of items on the stack
- -
-

Function memoization

-To reduce the number of calls to slow functions (such as $(shell) a single memoization function is provided.
-
-
memoize
-
-Arguments: 1: Name of function to memoize
-           2: String argument for the function
-Returns:   Result of $1 applied to $2 but only calls $1 once for each unique $2
- - -
-

Miscellaneous and Debugging Facilities

-GMSL defines the following constants; all are accessed as normal GNU -Make variables by wrapping them in $() or ${}.
-
- - - - - - - - - - - - - - - - - - - - - - - -
Constant
- 		Value
- 		Purpose
-
true
- 		T
- 		Boolean for $(if) -and return from  GMSL functions
-
false
-
- 		Boolean for $(if) -and return from GMSL functions
-
gmsl_version
- 		1 0 0
- 		GMSL version number as list: major minor revision
-
-
-gmsl_compatible
-
-Arguments: List containing the desired library version number (maj min -rev)
-Returns:   -$(true) if this version of the library is compatible
-           -with the requested version number, otherwise $(false) -
gmsl-print-% (target not a function)
-
-Arguments: The % should be -replaced by the name of a variable that you
-           wish to -print out.
-Action:    Echos the name of the variable that matches -the % and its value.
-           For -example, 'make gmsl-print-SHELL' will output the value of
-           the SHELL -variable
- -
gmsl-echo-% (target not a function)
-
-Arguments: The % should be -replaced by the name of a variable that you
-           wish to -print out.
-Action:    Echos the value of the variable that matches -the %.
-           For -example, 'make gmsl-echo-SHELL' will output the value of
-           the SHELL -variable
- -
assert
-
-Arguments: 1: A boolean that must -be true or the assertion will fail
-           2: The -message to print with the assertion
-Returns:   None
- -
assert_exists
-
-Arguments: 1: Name of file that -must exist, if it is missing an assertion
-           will be -generated
-Returns:   None
- -

-GMSL has a number of environment variables (or command-line overrides) -that control various bits of functionality:
-
- - - - - - - - - - - - - - - - - - - -
Variable
- 		Purpose
-
GMSL_NO_WARNINGS
- 		If set prevents GMSL from outputting warning messages: -artithmetic functions generate underflow warnings.
-
GMSL_NO_ERRORS
- 		If set prevents GMSL from generating fatal errors: division -by zero or failed assertions are fatal.
-
GMSL_TRACE
- 		Enables function tracing.  Calls to GMSL functions will -result in name and arguments being traced.
-
-
-
-Copyright (c) 2005-2022 John Graham-Cumming.
-
- diff --git a/mk/platform/unix_common.mk b/mk/platform/unix_common.mk index 8f06c9328..f611693f4 100644 --- a/mk/platform/unix_common.mk +++ b/mk/platform/unix_common.mk @@ -7,14 +7,8 @@ MKDIR=mkdir -p # Go should not be required to run functional tests GOOS ?= $(shell go env GOOS) -# Current versioning information from env +#Current versioning information from env BUILD_VERSION?=$(shell git describe --tags) BUILD_TIMESTAMP=$(shell date +%F"_"%T) DEFAULT_CONFIGDIR?=/etc/crowdsec DEFAULT_DATADIR?=/var/lib/crowdsec/data - -PKG_CONFIG:=$(shell command -v pkg-config 2>/dev/null) - -# See if we have libre2-dev installed for C++ optimizations. -# In fedora and other distros, we need to tell where to find re2.pc -RE2_CHECK := $(shell PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$(PKG_CONFIG_PATH) pkg-config --libs re2 2>/dev/null) diff --git a/mk/platform/windows.mk b/mk/platform/windows.mk index 68abc6853..8e2cdf19b 100644 --- a/mk/platform/windows.mk +++ b/mk/platform/windows.mk @@ -4,9 +4,9 @@ MAKE=make GOOS=windows PREFIX=$(shell $$env:TEMP) -# Current versioning information from env -# BUILD_VERSION?=$(shell (Invoke-WebRequest -UseBasicParsing -Uri https://api.github.com/repos/crowdsecurity/crowdsec/releases/latest).Content | jq -r '.tag_name') -# hardcode it till I find a workaround +#Current versioning information from env +#BUILD_VERSION?=$(shell (Invoke-WebRequest -UseBasicParsing -Uri https://api.github.com/repos/crowdsecurity/crowdsec/releases/latest).Content | jq -r '.tag_name') +#hardcode it till i find a workaround BUILD_VERSION?=$(shell git describe --tags $$(git rev-list --tags --max-count=1)) BUILD_TIMESTAMP?=$(shell Get-Date -Format "yyyy-MM-dd_HH:mm:ss") DEFAULT_CONFIGDIR?=C:\\ProgramData\\CrowdSec\\config @@ -18,5 +18,3 @@ CP=Copy-Item CPR=Copy-Item -Recurse MKDIR=New-Item -ItemType directory WIN_IGNORE_ERR=; exit 0 - -PKG_CONFIG:=$(shell Get-Command pkg-config -ErrorAction SilentlyContinue) diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index a7916dc2b..9fc8fc86f 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -35,20 +35,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" ) -type DataSourceUnavailableError struct { - Name string - Err error -} - -func (e *DataSourceUnavailableError) Error() string { - return fmt.Sprintf("datasource '%s' is not available: %v", e.Name, e.Err) -} - -func (e *DataSourceUnavailableError) Unwrap() error { - return e.Err -} - - // The interface each datasource must implement type DataSource interface { GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module @@ -89,10 +75,6 @@ func GetDataSourceIface(dataSourceType string) DataSource { return source() } -// DataSourceConfigure creates and returns a DataSource object from a configuration, -// if the configuration is not valid it returns an error. -// If the datasource can't be run (eg. journalctl not available), it still returns an error which -// can be checked for the appropriate action. func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataSource, error) { // we dump it back to []byte, because we want to decode the yaml blob twice: // once to DataSourceCommonCfg, and then later to the dedicated type of the datasource @@ -118,7 +100,7 @@ func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataS subLogger := clog.WithFields(customLog) /* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */ if err := dataSrc.CanRun(); err != nil { - return nil, &DataSourceUnavailableError{Name: commonConfig.Source, Err: err} + return nil, fmt.Errorf("datasource %s cannot be run: %w", commonConfig.Source, err) } /* configure the actual datasource */ if err := dataSrc.Configure(yamlConfig, subLogger); err != nil { @@ -191,11 +173,10 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, } dec := yaml.NewDecoder(yamlFile) dec.SetStrict(true) - idx := -1 for { var sub configuration.DataSourceCommonCfg + var idx int err = dec.Decode(&sub) - idx += 1 if err != nil { if !errors.Is(err, io.EOF) { return nil, fmt.Errorf("failed to yaml decode %s: %w", acquisFile, err) @@ -212,6 +193,7 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, if len(sub.Labels) == 0 { if sub.Source == "" { log.Debugf("skipping empty item in %s", acquisFile) + idx += 1 continue } return nil, fmt.Errorf("missing labels in %s (position: %d)", acquisFile, idx) @@ -226,11 +208,6 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, sub.UniqueId = uniqueId src, err := DataSourceConfigure(sub) if err != nil { - var dserr *DataSourceUnavailableError - if errors.As(err, &dserr) { - log.Error(err) - continue - } return nil, fmt.Errorf("while configuring datasource of type %s from %s (position: %d): %w", sub.Source, acquisFile, idx, err) } if sub.TransformExpr != "" { @@ -241,6 +218,7 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, transformRuntimes[uniqueId] = vm } sources = append(sources, *src) + idx += 1 } } return sources, nil @@ -317,11 +295,6 @@ func transform(transformChan chan types.Event, output chan types.Event, AcquisTo } func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb *tomb.Tomb) error { - // Don't wait if we have no sources, as it will hang forever - if len(sources) == 0 { - return nil - } - for i := 0; i < len(sources); i++ { subsrc := sources[i] //ensure its a copy log.Debugf("starting one source %d/%d ->> %T", i, len(sources), subsrc) @@ -357,8 +330,11 @@ func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb return nil }) } - - /*return only when acquisition is over (cat) or never (tail)*/ - err := AcquisTomb.Wait() - return err + // Don't wait if we have no sources, as it will hang forever + if len(sources) > 0 { + /*return only when acquisition is over (cat) or never (tail)*/ + err := AcquisTomb.Wait() + return err + } + return nil } diff --git a/pkg/acquisition/acquisition_test.go b/pkg/acquisition/acquisition_test.go index 548ecc04b..6b6d5ce71 100644 --- a/pkg/acquisition/acquisition_test.go +++ b/pkg/acquisition/acquisition_test.go @@ -167,7 +167,7 @@ log_level: debug source: mock_cant_run wowo: ajsajasjas `, - ExpectedError: "datasource 'mock_cant_run' is not available: can't run bro", + ExpectedError: "datasource mock_cant_run cannot be run: can't run bro", }, } diff --git a/pkg/acquisition/modules/cloudwatch/cloudwatch.go b/pkg/acquisition/modules/cloudwatch/cloudwatch.go index 48bbe4217..1abf04c5b 100644 --- a/pkg/acquisition/modules/cloudwatch/cloudwatch.go +++ b/pkg/acquisition/modules/cloudwatch/cloudwatch.go @@ -13,6 +13,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -200,7 +201,7 @@ func (cw *CloudwatchSource) Configure(yamlConfig []byte, logger *log.Entry) erro targetStream := "*" if cw.Config.StreamRegexp != nil { if _, err := regexp.Compile(*cw.Config.StreamRegexp); err != nil { - return fmt.Errorf("while compiling regexp '%s': %w", *cw.Config.StreamRegexp, err) + return errors.Wrapf(err, "error while compiling regexp '%s'", *cw.Config.StreamRegexp) } targetStream = *cw.Config.StreamRegexp } else if cw.Config.StreamName != nil { @@ -344,7 +345,8 @@ func (cw *CloudwatchSource) WatchLogGroupForStreams(out chan LogStreamTailConfig }, ) if err != nil { - return fmt.Errorf("while describing group %s: %w", cw.Config.GroupName, err) + newerr := errors.Wrapf(err, "while describing group %s", cw.Config.GroupName) + return newerr } cw.logger.Tracef("after DescribeLogStreamsPagesWithContext") } @@ -493,7 +495,7 @@ func (cw *CloudwatchSource) TailLogStream(cfg *LogStreamTailConfig, outChan chan }, ) if err != nil { - newerr := fmt.Errorf("while reading %s/%s: %w", cfg.GroupName, cfg.StreamName, err) + newerr := errors.Wrapf(err, "while reading %s/%s", cfg.GroupName, cfg.StreamName) cfg.logger.Warningf("err : %s", newerr) return newerr } @@ -530,7 +532,7 @@ func (cw *CloudwatchSource) ConfigureByDSN(dsn string, labels map[string]string, u, err := url.ParseQuery(args[1]) if err != nil { - return fmt.Errorf("while parsing %s: %w", dsn, err) + return errors.Wrapf(err, "while parsing %s", dsn) } for k, v := range u { @@ -541,7 +543,7 @@ func (cw *CloudwatchSource) ConfigureByDSN(dsn string, labels map[string]string, } lvl, err := log.ParseLevel(v[0]) if err != nil { - return fmt.Errorf("unknown level %s: %w", v[0], err) + return errors.Wrapf(err, "unknown level %s", v[0]) } cw.logger.Logger.SetLevel(lvl) @@ -575,7 +577,7 @@ func (cw *CloudwatchSource) ConfigureByDSN(dsn string, labels map[string]string, //let's reuse our parser helper so that a ton of date formats are supported duration, err := time.ParseDuration(v[0]) if err != nil { - return fmt.Errorf("unable to parse '%s' as duration: %w", v[0], err) + return errors.Wrapf(err, "unable to parse '%s' as duration", v[0]) } cw.logger.Debugf("parsed '%s' as '%s'", v[0], duration) start := time.Now().UTC().Add(-duration) @@ -672,7 +674,7 @@ func (cw *CloudwatchSource) CatLogStream(cfg *LogStreamTailConfig, outChan chan }, ) if err != nil { - return fmt.Errorf("while reading logs from %s/%s: %w", cfg.GroupName, cfg.StreamName, err) + return errors.Wrapf(err, "while reading logs from %s/%s", cfg.GroupName, cfg.StreamName) } cfg.logger.Tracef("after GetLogEventsPagesWithContext") case <-cw.t.Dying(): diff --git a/pkg/acquisition/modules/docker/docker.go b/pkg/acquisition/modules/docker/docker.go index 929626974..b1808e446 100644 --- a/pkg/acquisition/modules/docker/docker.go +++ b/pkg/acquisition/modules/docker/docker.go @@ -12,6 +12,7 @@ import ( dockerTypes "github.com/docker/docker/api/types" "github.com/docker/docker/client" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -79,7 +80,7 @@ func (d *DockerSource) UnmarshalConfig(yamlConfig []byte) error { err := yaml.UnmarshalStrict(yamlConfig, &d.Config) if err != nil { - return fmt.Errorf("while parsing DockerAcquisition configuration: %w", err) + return errors.Wrap(err, "Cannot parse DockerAcquisition configuration") } if d.logger != nil { @@ -213,7 +214,7 @@ func (d *DockerSource) ConfigureByDSN(dsn string, labels map[string]string, logg parameters, err := url.ParseQuery(args[1]) if err != nil { - return fmt.Errorf("while parsing parameters %s: %w", dsn, err) + return errors.Wrapf(err, "while parsing parameters %s: %s", dsn, err) } for k, v := range parameters { @@ -224,7 +225,7 @@ func (d *DockerSource) ConfigureByDSN(dsn string, labels map[string]string, logg } lvl, err := log.ParseLevel(v[0]) if err != nil { - return fmt.Errorf("unknown level %s: %w", v[0], err) + return errors.Wrapf(err, "unknown level %s", v[0]) } d.logger.Logger.SetLevel(lvl) case "until": diff --git a/pkg/acquisition/modules/file/file.go b/pkg/acquisition/modules/file/file.go index 799e78dca..c24b17332 100644 --- a/pkg/acquisition/modules/file/file.go +++ b/pkg/acquisition/modules/file/file.go @@ -14,6 +14,8 @@ import ( "strings" "time" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/fsnotify/fsnotify" "github.com/nxadm/tail" "github.com/pkg/errors" @@ -22,8 +24,6 @@ import ( "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" - "github.com/crowdsecurity/go-cs-lib/pkg/trace" - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -110,7 +110,7 @@ func (f *FileSource) Configure(yamlConfig []byte, logger *log.Entry) error { f.watcher, err = fsnotify.NewWatcher() if err != nil { - return fmt.Errorf("could not create fsnotify watcher: %w", err) + return errors.Wrapf(err, "Could not create fsnotify watcher") } f.logger.Tracef("Actual FileAcquisition Configuration %+v", f.config) @@ -130,7 +130,7 @@ func (f *FileSource) Configure(yamlConfig []byte, logger *log.Entry) error { } files, err := filepath.Glob(pattern) if err != nil { - return fmt.Errorf("glob failure: %w", err) + return errors.Wrap(err, "Glob failure") } if len(files) == 0 { f.logger.Warnf("No matching files for pattern %s", pattern) @@ -191,7 +191,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger if len(args) == 2 && len(args[1]) != 0 { params, err := url.ParseQuery(args[1]) if err != nil { - return fmt.Errorf("could not parse file args: %w", err) + return errors.Wrap(err, "could not parse file args") } for key, value := range params { switch key { @@ -201,7 +201,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger } lvl, err := log.ParseLevel(value[0]) if err != nil { - return fmt.Errorf("unknown level %s: %w", value[0], err) + return errors.Wrapf(err, "unknown level %s", value[0]) } f.logger.Logger.SetLevel(lvl) case "max_buffer_size": @@ -210,7 +210,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger } maxBufferSize, err := strconv.Atoi(value[0]) if err != nil { - return fmt.Errorf("could not parse max_buffer_size %s: %w", value[0], err) + return errors.Wrapf(err, "could not parse max_buffer_size %s", value[0]) } f.config.MaxBufferSize = maxBufferSize default: @@ -226,7 +226,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger f.logger.Debugf("Will try pattern %s", args[0]) files, err := filepath.Glob(args[0]) if err != nil { - return fmt.Errorf("glob failure: %w", err) + return errors.Wrap(err, "Glob failure") } if len(files) == 0 { @@ -433,7 +433,7 @@ func (f *FileSource) monitorNewFiles(out chan types.Event, t *tomb.Tomb) error { case <-t.Dying(): err := f.watcher.Close() if err != nil { - return fmt.Errorf("could not remove all inotify watches: %w", err) + return errors.Wrapf(err, "could not remove all inotify watches") } return nil } @@ -495,7 +495,7 @@ func (f *FileSource) readFile(filename string, out chan types.Event, t *tomb.Tom fd, err := os.Open(filename) if err != nil { - return fmt.Errorf("failed opening %s: %w", filename, err) + return errors.Wrapf(err, "failed opening %s", filename) } defer fd.Close() @@ -503,7 +503,7 @@ func (f *FileSource) readFile(filename string, out chan types.Event, t *tomb.Tom gz, err := gzip.NewReader(fd) if err != nil { logger.Errorf("Failed to read gz file: %s", err) - return fmt.Errorf("failed to read gz %s: %w", filename, err) + return errors.Wrapf(err, "failed to read gz %s", filename) } defer gz.Close() scanner = bufio.NewScanner(gz) diff --git a/pkg/acquisition/modules/file/file_test.go b/pkg/acquisition/modules/file/file_test.go index 8f80d050f..ff55bc413 100644 --- a/pkg/acquisition/modules/file/file_test.go +++ b/pkg/acquisition/modules/file/file_test.go @@ -38,7 +38,7 @@ func TestBadConfiguration(t *testing.T) { { name: "glob syntax error", config: `filename: "[asd-.log"`, - expectedErr: "glob failure: syntax error in pattern", + expectedErr: "Glob failure: syntax error in pattern", }, { name: "bad exclude regexp", @@ -150,7 +150,7 @@ filename: /`, config: ` mode: cat filename: "[*-.log"`, - expectedConfigErr: "glob failure: syntax error in pattern", + expectedConfigErr: "Glob failure: syntax error in pattern", logLevel: log.WarnLevel, expectedLines: 0, }, diff --git a/pkg/acquisition/modules/journalctl/journalctl.go b/pkg/acquisition/modules/journalctl/journalctl.go index b060ac364..7882cb7c2 100644 --- a/pkg/acquisition/modules/journalctl/journalctl.go +++ b/pkg/acquisition/modules/journalctl/journalctl.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -236,7 +237,7 @@ func (j *JournalCtlSource) ConfigureByDSN(dsn string, labels map[string]string, } lvl, err := log.ParseLevel(value[0]) if err != nil { - return fmt.Errorf("unknown level %s: %w", value[0], err) + return errors.Wrapf(err, "unknown level %s", value[0]) } j.logger.Logger.SetLevel(lvl) case "since": diff --git a/pkg/acquisition/modules/kafka/kafka.go b/pkg/acquisition/modules/kafka/kafka.go index dba8daf75..085751cfc 100644 --- a/pkg/acquisition/modules/kafka/kafka.go +++ b/pkg/acquisition/modules/kafka/kafka.go @@ -10,6 +10,7 @@ import ( "strconv" "time" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/segmentio/kafka-go" log "github.com/sirupsen/logrus" @@ -92,12 +93,12 @@ func (k *KafkaSource) Configure(yamlConfig []byte, logger *log.Entry) error { dialer, err := k.Config.NewDialer() if err != nil { - return fmt.Errorf("cannot create %s dialer: %w", dataSourceName, err) + return errors.Wrapf(err, "cannot create %s dialer", dataSourceName) } k.Reader, err = k.Config.NewReader(dialer) if err != nil { - return fmt.Errorf("cannote create %s reader: %w", dataSourceName, err) + return errors.Wrapf(err, "cannote create %s reader", dataSourceName) } if k.Reader == nil { @@ -148,7 +149,7 @@ func (k *KafkaSource) ReadMessage(out chan types.Event) error { if err == io.EOF { return nil } - k.logger.Errorln(fmt.Errorf("while reading %s message: %w", dataSourceName, err)) + k.logger.Errorln(errors.Wrapf(err, "while reading %s message", dataSourceName)) } l := types.Line{ Raw: string(m.Value), @@ -180,7 +181,7 @@ func (k *KafkaSource) RunReader(out chan types.Event, t *tomb.Tomb) error { case <-t.Dying(): k.logger.Infof("%s datasource topic %s stopping", dataSourceName, k.Config.Topic) if err := k.Reader.Close(); err != nil { - return fmt.Errorf("while closing %s reader on topic '%s': %w", dataSourceName, k.Config.Topic, err) + return errors.Wrapf(err, "while closing %s reader on topic '%s'", dataSourceName, k.Config.Topic) } return nil } @@ -263,7 +264,7 @@ func (kc *KafkaConfiguration) NewReader(dialer *kafka.Dialer) (*kafka.Reader, er rConf.GroupID = kc.GroupID } if err := rConf.Validate(); err != nil { - return &kafka.Reader{}, fmt.Errorf("while validating reader configuration: %w", err) + return &kafka.Reader{}, errors.Wrapf(err, "while validating reader configuration") } return kafka.NewReader(rConf), nil } diff --git a/pkg/acquisition/modules/kafka/kafka_test.go b/pkg/acquisition/modules/kafka/kafka_test.go index 950d33a62..b37d0e7b7 100644 --- a/pkg/acquisition/modules/kafka/kafka_test.go +++ b/pkg/acquisition/modules/kafka/kafka_test.go @@ -8,14 +8,13 @@ import ( "testing" "time" - "github.com/segmentio/kafka-go" - log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/require" - "gopkg.in/tomb.v2" - "github.com/crowdsecurity/go-cs-lib/pkg/cstest" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/segmentio/kafka-go" + log "github.com/sirupsen/logrus" + "gopkg.in/tomb.v2" + "gotest.tools/v3/assert" ) func TestConfigure(t *testing.T) { @@ -179,7 +178,7 @@ topic: crowdsecplaintext`), subLogger) break READLOOP } } - require.Equal(t, ts.expectedLines, actualLines) + assert.Equal(t, ts.expectedLines, actualLines) tomb.Kill(nil) tomb.Wait() }) @@ -255,7 +254,7 @@ tls: break READLOOP } } - require.Equal(t, ts.expectedLines, actualLines) + assert.Equal(t, ts.expectedLines, actualLines) tomb.Kill(nil) tomb.Wait() }) diff --git a/pkg/acquisition/modules/kinesis/kinesis.go b/pkg/acquisition/modules/kinesis/kinesis.go index cc263da4f..60cdc3751 100644 --- a/pkg/acquisition/modules/kinesis/kinesis.go +++ b/pkg/acquisition/modules/kinesis/kinesis.go @@ -13,6 +13,7 @@ import ( "github.com/aws/aws-sdk-go/aws/arn" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/kinesis" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -123,7 +124,7 @@ func (k *KinesisSource) UnmarshalConfig(yamlConfig []byte) error { err := yaml.UnmarshalStrict(yamlConfig, &k.Config) if err != nil { - return fmt.Errorf("Cannot parse kinesis datasource configuration: %w", err) + return errors.Wrap(err, "Cannot parse kinesis datasource configuration") } if k.Config.Mode == "" { @@ -217,7 +218,7 @@ func (k *KinesisSource) WaitForConsumerDeregistration(consumerName string, strea return nil default: k.logger.Errorf("Error while waiting for consumer deregistration: %s", err) - return fmt.Errorf("cannot describe stream consumer: %w", err) + return errors.Wrap(err, "Cannot describe stream consumer") } } time.Sleep(time.Millisecond * 200 * time.Duration(i+1)) @@ -235,12 +236,12 @@ func (k *KinesisSource) DeregisterConsumer() error { switch err.(type) { case *kinesis.ResourceNotFoundException: default: - return fmt.Errorf("cannot deregister stream consumer: %w", err) + return errors.Wrap(err, "Cannot deregister stream consumer") } } err = k.WaitForConsumerDeregistration(k.Config.ConsumerName, k.Config.StreamARN) if err != nil { - return fmt.Errorf("cannot wait for consumer deregistration: %w", err) + return errors.Wrap(err, "Cannot wait for consumer deregistration") } return nil } @@ -252,7 +253,7 @@ func (k *KinesisSource) WaitForConsumerRegistration(consumerARN string) error { ConsumerARN: aws.String(consumerARN), }) if err != nil { - return fmt.Errorf("cannot describe stream consumer: %w", err) + return errors.Wrap(err, "Cannot describe stream consumer") } if *describeOutput.ConsumerDescription.ConsumerStatus == "ACTIVE" { k.logger.Debugf("Consumer %s is active", consumerARN) @@ -271,11 +272,11 @@ func (k *KinesisSource) RegisterConsumer() (*kinesis.RegisterStreamConsumerOutpu StreamARN: aws.String(k.Config.StreamARN), }) if err != nil { - return nil, fmt.Errorf("cannot register stream consumer: %w", err) + return nil, errors.Wrap(err, "Cannot register stream consumer") } err = k.WaitForConsumerRegistration(*streamConsumer.Consumer.ConsumerARN) if err != nil { - return nil, fmt.Errorf("timeout while waiting for consumer to be active: %w", err) + return nil, errors.Wrap(err, "Timeout while waiting for consumer to be active") } return streamConsumer, nil } @@ -338,7 +339,7 @@ func (k *KinesisSource) ReadFromSubscription(reader kinesis.SubscribeToShardEven logger.Infof("Subscribed shard reader is dying") err := reader.Close() if err != nil { - return fmt.Errorf("cannot close kinesis subscribed shard reader: %w", err) + return errors.Wrap(err, "Cannot close kinesis subscribed shard reader") } return nil case event, ok := <-reader.Events(): @@ -361,7 +362,7 @@ func (k *KinesisSource) SubscribeToShards(arn arn.ARN, streamConsumer *kinesis.R StreamName: aws.String(arn.Resource[7:]), }) if err != nil { - return fmt.Errorf("cannot list shards for enhanced_read: %w", err) + return errors.Wrap(err, "Cannot list shards for enhanced_read") } for _, shard := range shards.Shards { @@ -372,7 +373,7 @@ func (k *KinesisSource) SubscribeToShards(arn arn.ARN, streamConsumer *kinesis.R ConsumerARN: streamConsumer.Consumer.ConsumerARN, }) if err != nil { - return fmt.Errorf("cannot subscribe to shard: %w", err) + return errors.Wrap(err, "Cannot subscribe to shard") } k.shardReaderTomb.Go(func() error { return k.ReadFromSubscription(r.GetEventStream().Reader, out, shardId, arn.Resource[7:]) @@ -384,7 +385,7 @@ func (k *KinesisSource) SubscribeToShards(arn arn.ARN, streamConsumer *kinesis.R func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error { parsedARN, err := arn.Parse(k.Config.StreamARN) if err != nil { - return fmt.Errorf("cannot parse stream ARN: %w", err) + return errors.Wrap(err, "Cannot parse stream ARN") } if !strings.HasPrefix(parsedARN.Resource, "stream/") { return fmt.Errorf("resource part of stream ARN %s does not start with stream/", k.Config.StreamARN) @@ -394,12 +395,12 @@ func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error { k.logger.Info("starting kinesis acquisition with enhanced fan-out") err = k.DeregisterConsumer() if err != nil { - return fmt.Errorf("cannot deregister consumer: %w", err) + return errors.Wrap(err, "Cannot deregister consumer") } streamConsumer, err := k.RegisterConsumer() if err != nil { - return fmt.Errorf("cannot register consumer: %w", err) + return errors.Wrap(err, "Cannot register consumer") } for { @@ -407,7 +408,7 @@ func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error { err = k.SubscribeToShards(parsedARN, streamConsumer, out) if err != nil { - return fmt.Errorf("cannot subscribe to shards: %w", err) + return errors.Wrap(err, "Cannot subscribe to shards") } select { case <-t.Dying(): @@ -416,7 +417,7 @@ func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error { _ = k.shardReaderTomb.Wait() //we don't care about the error as we kill the tomb ourselves err = k.DeregisterConsumer() if err != nil { - return fmt.Errorf("cannot deregister consumer: %w", err) + return errors.Wrap(err, "Cannot deregister consumer") } return nil case <-k.shardReaderTomb.Dying(): @@ -439,7 +440,7 @@ func (k *KinesisSource) ReadFromShard(out chan types.Event, shardId string) erro ShardIteratorType: aws.String(kinesis.ShardIteratorTypeLatest)}) if err != nil { logger.Errorf("Cannot get shard iterator: %s", err) - return fmt.Errorf("cannot get shard iterator: %w", err) + return errors.Wrap(err, "Cannot get shard iterator") } it := sharIt.ShardIterator //AWS recommends to wait for a second between calls to GetRecords for a given shard @@ -460,7 +461,7 @@ func (k *KinesisSource) ReadFromShard(out chan types.Event, shardId string) erro continue default: logger.Error("Cannot get records") - return fmt.Errorf("cannot get records: %w", err) + return errors.Wrap(err, "Cannot get records") } } k.ParseAndPushRecords(records.Records, out, logger, shardId) @@ -485,7 +486,7 @@ func (k *KinesisSource) ReadFromStream(out chan types.Event, t *tomb.Tomb) error StreamName: aws.String(k.Config.StreamName), }) if err != nil { - return fmt.Errorf("cannot list shards: %w", err) + return errors.Wrap(err, "Cannot list shards") } k.shardReaderTomb = &tomb.Tomb{} for _, shard := range shards.Shards { diff --git a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go index 243547381..f65a0aa57 100644 --- a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go +++ b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go @@ -8,16 +8,16 @@ import ( "net/http" "strings" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" "k8s.io/apiserver/pkg/apis/audit" - - "github.com/crowdsecurity/go-cs-lib/pkg/trace" - - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" - "github.com/crowdsecurity/crowdsec/pkg/types" ) type KubernetesAuditConfiguration struct { @@ -66,7 +66,7 @@ func (ka *KubernetesAuditSource) UnmarshalConfig(yamlConfig []byte) error { k8sConfig := KubernetesAuditConfiguration{} err := yaml.UnmarshalStrict(yamlConfig, &k8sConfig) if err != nil { - return fmt.Errorf("cannot parse k8s-audit configuration: %w", err) + return errors.Wrap(err, "Cannot parse k8s-audit configuration") } ka.config = k8sConfig @@ -140,7 +140,7 @@ func (ka *KubernetesAuditSource) StreamingAcquisition(out chan types.Event, t *t t.Go(func() error { err := ka.server.ListenAndServe() if err != nil && err != http.ErrServerClosed { - return fmt.Errorf("k8s-audit server failed: %w", err) + return errors.Wrap(err, "k8s-audit server failed") } return nil }) diff --git a/pkg/acquisition/modules/s3/s3.go b/pkg/acquisition/modules/s3/s3.go index 96026f55e..4ba31f43b 100644 --- a/pkg/acquisition/modules/s3/s3.go +++ b/pkg/acquisition/modules/s3/s3.go @@ -6,7 +6,6 @@ import ( "compress/gzip" "context" "encoding/json" - "errors" "fmt" "io" "net/url" @@ -22,13 +21,13 @@ import ( "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/aws/aws-sdk-go/service/sqs" "github.com/aws/aws-sdk-go/service/sqs/sqsiface" + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" - - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" - "github.com/crowdsecurity/crowdsec/pkg/types" ) type S3Configuration struct { @@ -564,7 +563,7 @@ func (s *S3Source) ConfigureByDSN(dsn string, labels map[string]string, logger * if len(args) == 2 && len(args[1]) != 0 { params, err := url.ParseQuery(args[1]) if err != nil { - return fmt.Errorf("could not parse s3 args: %w", err) + return errors.Wrap(err, "could not parse s3 args") } for key, value := range params { switch key { @@ -574,7 +573,7 @@ func (s *S3Source) ConfigureByDSN(dsn string, labels map[string]string, logger * } lvl, err := log.ParseLevel(value[0]) if err != nil { - return fmt.Errorf("unknown level %s: %w", value[0], err) + return errors.Wrapf(err, "unknown level %s", value[0]) } s.logger.Logger.SetLevel(lvl) case "max_buffer_size": @@ -583,7 +582,7 @@ func (s *S3Source) ConfigureByDSN(dsn string, labels map[string]string, logger * } maxBufferSize, err := strconv.Atoi(value[0]) if err != nil { - return fmt.Errorf("invalid value for 'max_buffer_size': %w", err) + return errors.Wrapf(err, "invalid value for 'max_buffer_size'") } s.logger.Debugf("Setting max buffer size to %d", maxBufferSize) s.Config.MaxBufferSize = maxBufferSize diff --git a/pkg/acquisition/modules/syslog/internal/server/syslogserver.go b/pkg/acquisition/modules/syslog/internal/server/syslogserver.go index 7118c295b..088ab0d95 100644 --- a/pkg/acquisition/modules/syslog/internal/server/syslogserver.go +++ b/pkg/acquisition/modules/syslog/internal/server/syslogserver.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" ) @@ -30,18 +31,18 @@ func (s *SyslogServer) Listen(listenAddr string, port int) error { s.port = port udpAddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", s.listenAddr, s.port)) if err != nil { - return fmt.Errorf("could not resolve addr %s: %w", s.listenAddr, err) + return errors.Wrapf(err, "could not resolve addr %s", s.listenAddr) } udpConn, err := net.ListenUDP("udp", udpAddr) if err != nil { - return fmt.Errorf("could not listen on port %d: %w", s.port, err) + return errors.Wrapf(err, "could not listen on port %d", s.port) } s.Logger.Debugf("listening on %s:%d", s.listenAddr, s.port) s.udpConn = udpConn err = s.udpConn.SetReadDeadline(time.Now().UTC().Add(100 * time.Millisecond)) if err != nil { - return fmt.Errorf("could not set read deadline on UDP socket: %w", err) + return errors.Wrap(err, "could not set read deadline on UDP socket") } return nil } @@ -86,7 +87,7 @@ func (s *SyslogServer) StartServer() *tomb.Tomb { func (s *SyslogServer) KillServer() error { err := s.udpConn.Close() if err != nil { - return fmt.Errorf("could not close UDP connection: %w", err) + return errors.Wrap(err, "could not close UDP connection") } close(s.channel) return nil diff --git a/pkg/acquisition/modules/syslog/syslog.go b/pkg/acquisition/modules/syslog/syslog.go index 840e37200..948f3d005 100644 --- a/pkg/acquisition/modules/syslog/syslog.go +++ b/pkg/acquisition/modules/syslog/syslog.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -99,7 +100,7 @@ func (s *SyslogSource) UnmarshalConfig(yamlConfig []byte) error { err := yaml.UnmarshalStrict(yamlConfig, &s.config) if err != nil { - return fmt.Errorf("cannot parse syslog configuration: %w", err) + return errors.Wrap(err, "Cannot parse syslog configuration") } if s.config.Addr == "" { @@ -139,7 +140,7 @@ func (s *SyslogSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) s.server.SetChannel(c) err := s.server.Listen(s.config.Addr, s.config.Port) if err != nil { - return fmt.Errorf("could not start syslog server: %w", err) + return errors.Wrap(err, "could not start syslog server") } s.serverTomb = s.server.StartServer() t.Go(func() error { diff --git a/pkg/apiclient/alerts_service.go b/pkg/apiclient/alerts_service.go index dd2ba2975..b8d1b7fc1 100644 --- a/pkg/apiclient/alerts_service.go +++ b/pkg/apiclient/alerts_service.go @@ -5,9 +5,9 @@ import ( "fmt" "net/http" - qs "github.com/google/go-querystring/query" - "github.com/crowdsecurity/crowdsec/pkg/models" + qs "github.com/google/go-querystring/query" + "github.com/pkg/errors" ) // type ApiAlerts service @@ -72,7 +72,7 @@ func (s *AlertsService) List(ctx context.Context, opts AlertsListOpts) (*models. u := fmt.Sprintf("%s/alerts", s.client.URLPrefix) params, err := qs.Values(opts) if err != nil { - return nil, nil, fmt.Errorf("building query: %w", err) + return nil, nil, errors.Wrap(err, "building query") } if len(params) > 0 { URI = fmt.Sprintf("%s?%s", u, params.Encode()) @@ -82,12 +82,12 @@ func (s *AlertsService) List(ctx context.Context, opts AlertsListOpts) (*models. req, err := s.client.NewRequest(http.MethodGet, URI, nil) if err != nil { - return nil, nil, fmt.Errorf("building request: %w", err) + return nil, nil, errors.Wrap(err, "building request") } resp, err := s.client.Do(ctx, req, &alerts) if err != nil { - return nil, resp, fmt.Errorf("performing request: %w", err) + return nil, resp, errors.Wrap(err, "performing request") } return &alerts, resp, nil } diff --git a/pkg/apiclient/alerts_service_test.go b/pkg/apiclient/alerts_service_test.go index aa5039f0b..f4ec8cabe 100644 --- a/pkg/apiclient/alerts_service_test.go +++ b/pkg/apiclient/alerts_service_test.go @@ -8,13 +8,12 @@ import ( "reflect" "testing" - log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/crowdsecurity/go-cs-lib/pkg/version" "github.com/crowdsecurity/crowdsec/pkg/models" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestAlertsListAsMachine(t *testing.T) { diff --git a/pkg/apiclient/auth.go b/pkg/apiclient/auth.go index 84df74456..48b971a06 100644 --- a/pkg/apiclient/auth.go +++ b/pkg/apiclient/auth.go @@ -3,21 +3,23 @@ package apiclient import ( "bytes" "encoding/json" - "fmt" - "io" "math/rand" - "net/http" - "net/http/httputil" - "net/url" "sync" "time" - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" + //"errors" + "fmt" + "io" + "net/http" + "net/http/httputil" + "net/url" "github.com/crowdsecurity/crowdsec/pkg/fflag" "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/go-openapi/strfmt" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + //"google.golang.org/appengine/log" ) type APIKeyTransport struct { @@ -167,11 +169,11 @@ func (t *JWTTransport) refreshJwtToken() error { enc.SetEscapeHTML(false) err = enc.Encode(auth) if err != nil { - return fmt.Errorf("could not encode jwt auth body: %w", err) + return errors.Wrap(err, "could not encode jwt auth body") } req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s%s/watchers/login", t.URL, t.VersionPrefix), buf) if err != nil { - return fmt.Errorf("could not create request: %w", err) + return errors.Wrap(err, "could not create request") } req.Header.Add("Content-Type", "application/json") client := &http.Client{ @@ -194,7 +196,7 @@ func (t *JWTTransport) refreshJwtToken() error { resp, err := client.Do(req) if err != nil { - return fmt.Errorf("could not get jwt token: %w", err) + return errors.Wrap(err, "could not get jwt token") } log.Debugf("auth-jwt : http %d", resp.StatusCode) @@ -215,10 +217,10 @@ func (t *JWTTransport) refreshJwtToken() error { } if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { - return fmt.Errorf("unable to decode response: %w", err) + return errors.Wrap(err, "unable to decode response") } if err := t.Expiration.UnmarshalText([]byte(response.Expire)); err != nil { - return fmt.Errorf("unable to parse jwt expiration: %w", err) + return errors.Wrap(err, "unable to parse jwt expiration") } t.Token = response.Token @@ -261,7 +263,7 @@ func (t *JWTTransport) RoundTrip(req *http.Request) (*http.Response, error) { if err != nil { /*we had an error (network error for example, or 401 because token is refused), reset the token ?*/ t.Token = "" - return resp, fmt.Errorf("performing jwt auth: %w", err) + return resp, errors.Wrapf(err, "performing jwt auth") } log.Debugf("resp-jwt: %d", resp.StatusCode) diff --git a/pkg/apiclient/auth_service.go b/pkg/apiclient/auth_service.go index 64284902e..26ad80c0c 100644 --- a/pkg/apiclient/auth_service.go +++ b/pkg/apiclient/auth_service.go @@ -21,6 +21,7 @@ type enrollRequest struct { } func (s *AuthService) UnregisterWatcher(ctx context.Context) (*Response, error) { + u := fmt.Sprintf("%s/watchers", s.client.URLPrefix) req, err := s.client.NewRequest(http.MethodDelete, u, nil) if err != nil { @@ -35,6 +36,7 @@ func (s *AuthService) UnregisterWatcher(ctx context.Context) (*Response, error) } func (s *AuthService) RegisterWatcher(ctx context.Context, registration models.WatcherRegistrationRequest) (*Response, error) { + u := fmt.Sprintf("%s/watchers", s.client.URLPrefix) req, err := s.client.NewRequest(http.MethodPost, u, ®istration) diff --git a/pkg/apiclient/auth_service_test.go b/pkg/apiclient/auth_service_test.go index 32ba1890f..6236cf041 100644 --- a/pkg/apiclient/auth_service_test.go +++ b/pkg/apiclient/auth_service_test.go @@ -10,12 +10,11 @@ import ( "net/url" "testing" - log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/crowdsecurity/go-cs-lib/pkg/version" "github.com/crowdsecurity/crowdsec/pkg/models" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" ) type BasicMockPayload struct { diff --git a/pkg/apiclient/client.go b/pkg/apiclient/client.go index d95f77490..a7db1ed33 100644 --- a/pkg/apiclient/client.go +++ b/pkg/apiclient/client.go @@ -11,6 +11,7 @@ import ( "net/url" "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/pkg/errors" ) var ( @@ -124,9 +125,9 @@ func RegisterClient(config *Config, client *http.Client) (*ApiClient, error) { /*if we have http status, return it*/ if err != nil { if resp != nil && resp.Response != nil { - return nil, fmt.Errorf("api register (%s) http %s: %w", c.BaseURL, resp.Response.Status, err) + return nil, errors.Wrapf(err, "api register (%s) http %s : %s", c.BaseURL, resp.Response.Status, err) } - return nil, fmt.Errorf("api register (%s): %w", c.BaseURL, err) + return nil, errors.Wrapf(err, "api register (%s) : %s", c.BaseURL, err) } return c, nil @@ -165,7 +166,7 @@ func CheckResponse(r *http.Response) error { if err == nil && data != nil { err := json.Unmarshal(data, errorResponse) if err != nil { - return fmt.Errorf("http code %d, invalid body: %w", r.StatusCode, err) + return errors.Wrapf(err, "http code %d, invalid body", r.StatusCode) } } else { errorResponse.Message = new(string) diff --git a/pkg/apiclient/client_http_test.go b/pkg/apiclient/client_http_test.go index 9e082cf51..c50769041 100644 --- a/pkg/apiclient/client_http_test.go +++ b/pkg/apiclient/client_http_test.go @@ -8,9 +8,9 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" - "github.com/crowdsecurity/go-cs-lib/pkg/version" + + "github.com/stretchr/testify/assert" ) func TestNewRequestInvalid(t *testing.T) { diff --git a/pkg/apiclient/client_test.go b/pkg/apiclient/client_test.go index 08f56730b..ef52a60ab 100644 --- a/pkg/apiclient/client_test.go +++ b/pkg/apiclient/client_test.go @@ -9,9 +9,10 @@ import ( "runtime" "testing" - log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/go-cs-lib/pkg/version" ) diff --git a/pkg/apiclient/decisions_service.go b/pkg/apiclient/decisions_service.go index e96394f56..054c51a9c 100644 --- a/pkg/apiclient/decisions_service.go +++ b/pkg/apiclient/decisions_service.go @@ -6,15 +6,14 @@ import ( "fmt" "net/http" - qs "github.com/google/go-querystring/query" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/go-cs-lib/pkg/ptr" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/modelscapi" "github.com/crowdsecurity/crowdsec/pkg/types" + qs "github.com/google/go-querystring/query" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) type DecisionsService service diff --git a/pkg/apiclient/decisions_service_test.go b/pkg/apiclient/decisions_service_test.go index ab7e46e64..935ddcea5 100644 --- a/pkg/apiclient/decisions_service_test.go +++ b/pkg/apiclient/decisions_service_test.go @@ -8,15 +8,14 @@ import ( "reflect" "testing" - log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/crowdsecurity/go-cs-lib/pkg/ptr" "github.com/crowdsecurity/go-cs-lib/pkg/version" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/modelscapi" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestDecisionsList(t *testing.T) { diff --git a/pkg/apiclient/decisions_sync_service.go b/pkg/apiclient/decisions_sync_service.go index 57999691f..e5284f4d9 100644 --- a/pkg/apiclient/decisions_sync_service.go +++ b/pkg/apiclient/decisions_sync_service.go @@ -5,9 +5,9 @@ import ( "fmt" "net/http" - log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) type DecisionDeleteService service @@ -18,12 +18,12 @@ func (d *DecisionDeleteService) Add(ctx context.Context, deletedDecisions *model u := fmt.Sprintf("%s/decisions/delete", d.client.URLPrefix) req, err := d.client.NewRequest(http.MethodPost, u, &deletedDecisions) if err != nil { - return nil, nil, fmt.Errorf("while building request: %w", err) + return nil, nil, errors.Wrap(err, "while building request") } resp, err := d.client.Do(ctx, req, &response) if err != nil { - return nil, resp, fmt.Errorf("while performing request: %w", err) + return nil, resp, errors.Wrap(err, "while performing request") } if resp.Response.StatusCode != http.StatusOK { log.Warnf("Decisions delete response : http %s", resp.Response.Status) diff --git a/pkg/apiclient/signal.go b/pkg/apiclient/signal.go index 2dceb8157..27d2e3693 100644 --- a/pkg/apiclient/signal.go +++ b/pkg/apiclient/signal.go @@ -8,6 +8,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/pkg/errors" ) type SignalService service @@ -18,12 +19,12 @@ func (s *SignalService) Add(ctx context.Context, signals *models.AddSignalsReque u := fmt.Sprintf("%s/signals", s.client.URLPrefix) req, err := s.client.NewRequest(http.MethodPost, u, &signals) if err != nil { - return nil, nil, fmt.Errorf("while building request: %w", err) + return nil, nil, errors.Wrap(err, "while building request") } resp, err := s.client.Do(ctx, req, &response) if err != nil { - return nil, resp, fmt.Errorf("while performing request: %w", err) + return nil, resp, errors.Wrap(err, "while performing request") } if resp.Response.StatusCode != http.StatusOK { log.Warnf("Signal push response : http %s", resp.Response.Status) diff --git a/pkg/apiserver/apic.go b/pkg/apiserver/apic.go index b15cf21d6..3791a9a3c 100644 --- a/pkg/apiserver/apic.go +++ b/pkg/apiserver/apic.go @@ -33,8 +33,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" ) -const ( - // delta values must be smaller than the interval +var ( pullIntervalDefault = time.Hour * 2 pullIntervalDelta = 5 * time.Minute pushIntervalDefault = time.Second * 10 @@ -72,12 +71,7 @@ type apic struct { // randomDuration returns a duration value between d-delta and d+delta func randomDuration(d time.Duration, delta time.Duration) time.Duration { - ret := d + time.Duration(rand.Int63n(int64(2*delta))) - delta - // ticker interval must be > 0 (nanoseconds) - if ret <= 0 { - return 1 - } - return ret + return time.Duration(float64(d) + float64(delta)*(-1.0+2.0*rand.Float64())) } func (a *apic) FetchScenariosListFromDB() ([]string, error) { @@ -828,6 +822,80 @@ func (a *apic) Pull() error { } } +func (a *apic) GetMetrics() (*models.Metrics, error) { + metric := &models.Metrics{ + ApilVersion: ptr.Of(version.String()), + Machines: make([]*models.MetricsAgentInfo, 0), + Bouncers: make([]*models.MetricsBouncerInfo, 0), + } + machines, err := a.dbClient.ListMachines() + if err != nil { + return metric, err + } + bouncers, err := a.dbClient.ListBouncers() + if err != nil { + return metric, err + } + var lastpush string + for _, machine := range machines { + if machine.LastPush == nil { + lastpush = time.Time{}.String() + } else { + lastpush = machine.LastPush.String() + } + m := &models.MetricsAgentInfo{ + Version: machine.Version, + Name: machine.MachineId, + LastUpdate: machine.UpdatedAt.String(), + LastPush: lastpush, + } + metric.Machines = append(metric.Machines, m) + } + + for _, bouncer := range bouncers { + m := &models.MetricsBouncerInfo{ + Version: bouncer.Version, + CustomName: bouncer.Name, + Name: bouncer.Type, + LastPull: bouncer.LastPull.String(), + } + metric.Bouncers = append(metric.Bouncers, m) + } + return metric, nil +} + +func (a *apic) SendMetrics(stop chan (bool)) { + defer trace.CatchPanic("lapi/metricsToAPIC") + + ticker := time.NewTicker(a.metricsIntervalFirst) + + log.Infof("Start send metrics to CrowdSec Central API (interval: %s once, then %s)", a.metricsIntervalFirst.Round(time.Second), a.metricsInterval) + + for { + metrics, err := a.GetMetrics() + if err != nil { + log.Errorf("unable to get metrics (%s), will retry", err) + } + _, _, err = a.apiClient.Metrics.Add(context.Background(), metrics) + if err != nil { + log.Errorf("capi metrics: failed: %s", err) + } else { + log.Infof("capi metrics: metrics sent successfully") + } + + select { + case <-stop: + return + case <-ticker.C: + ticker.Reset(a.metricsInterval) + case <-a.metricsTomb.Dying(): // if one apic routine is dying, do we kill the others? + a.pullTomb.Kill(nil) + a.pushTomb.Kill(nil) + return + } + } +} + func (a *apic) Shutdown() { a.pushTomb.Kill(nil) a.pullTomb.Kill(nil) diff --git a/pkg/apiserver/apic_metrics.go b/pkg/apiserver/apic_metrics.go deleted file mode 100644 index 4befcf50c..000000000 --- a/pkg/apiserver/apic_metrics.go +++ /dev/null @@ -1,145 +0,0 @@ -package apiserver - -import ( - "context" - "time" - - log "github.com/sirupsen/logrus" - "golang.org/x/exp/slices" - - "github.com/crowdsecurity/go-cs-lib/pkg/ptr" - "github.com/crowdsecurity/go-cs-lib/pkg/trace" - "github.com/crowdsecurity/go-cs-lib/pkg/version" - - "github.com/crowdsecurity/crowdsec/pkg/models" -) - -func (a *apic) GetMetrics() (*models.Metrics, error) { - machines, err := a.dbClient.ListMachines() - if err != nil { - return nil, err - } - - machinesInfo := make([]*models.MetricsAgentInfo, len(machines)) - - for i, machine := range machines { - machinesInfo[i] = &models.MetricsAgentInfo{ - Version: machine.Version, - Name: machine.MachineId, - LastUpdate: machine.UpdatedAt.String(), - LastPush: ptr.OrEmpty(machine.LastPush).String(), - } - } - - bouncers, err := a.dbClient.ListBouncers() - if err != nil { - return nil, err - } - - bouncersInfo := make([]*models.MetricsBouncerInfo, len(bouncers)) - - for i, bouncer := range bouncers { - bouncersInfo[i] = &models.MetricsBouncerInfo{ - Version: bouncer.Version, - CustomName: bouncer.Name, - Name: bouncer.Type, - LastPull: bouncer.LastPull.String(), - } - } - - return &models.Metrics{ - ApilVersion: ptr.Of(version.String()), - Machines: machinesInfo, - Bouncers: bouncersInfo, - }, nil -} - -func (a *apic) fetchMachineIDs() ([]string, error) { - machines, err := a.dbClient.ListMachines() - if err != nil { - return nil, err - } - - ret := make([]string, len(machines)) - for i, machine := range machines { - ret[i] = machine.MachineId - } - // sorted slices are required for the slices.Equal comparison - slices.Sort(ret) - return ret, nil -} - -// SendMetrics sends metrics to the API server until it receives a stop signal. -// -// Metrics are sent at start, then at the randomized metricsIntervalFirst, -// then at regular metricsInterval. If a change is detected in the list -// of machines, the next metrics are sent immediately. -func (a *apic) SendMetrics(stop chan (bool)) { - defer trace.CatchPanic("lapi/metricsToAPIC") - - // verify the list of machines every interval - const checkInt = 20 * time.Second - - // intervals must always be > 0 - metInts := []time.Duration{1, a.metricsIntervalFirst, a.metricsInterval} - - log.Infof("Start send metrics to CrowdSec Central API (interval: %s once, then %s)", - metInts[1].Round(time.Second), metInts[2]) - - count := -1 - nextMetInt := func() time.Duration { - if count < len(metInts)-1 { - count++ - } - return metInts[count] - } - - // store the list of machine IDs to compare - // with the next list - machineIDs := []string{} - - reloadMachineIDs := func() { - ids, err := a.fetchMachineIDs() - if err != nil { - log.Debugf("unable to get machines (%s), will retry", err) - return - } - machineIDs = ids - } - - checkTicker := time.NewTicker(checkInt) - metTicker := time.NewTicker(nextMetInt()) - - for { - select { - case <-stop: - checkTicker.Stop() - metTicker.Stop() - return - case <-checkTicker.C: - oldIDs := machineIDs - reloadMachineIDs() - if !slices.Equal(oldIDs, machineIDs) { - log.Infof("capi metrics: machines changed, immediate send") - metTicker.Reset(1) - } - case <-metTicker.C: - metrics, err := a.GetMetrics() - if err != nil { - log.Errorf("unable to get metrics (%s), will retry", err) - } - log.Info("capi metrics: sending") - _, _, err = a.apiClient.Metrics.Add(context.Background(), metrics) - if err != nil { - log.Errorf("capi metrics: failed: %s", err) - } - metTicker.Reset(nextMetInt()) - case <-a.metricsTomb.Dying(): // if one apic routine is dying, do we kill the others? - checkTicker.Stop() - metTicker.Stop() - a.pullTomb.Kill(nil) - a.pushTomb.Kill(nil) - return - } - } -} diff --git a/pkg/apiserver/apic_metrics_test.go b/pkg/apiserver/apic_metrics_test.go deleted file mode 100644 index 7e37ea1e9..000000000 --- a/pkg/apiserver/apic_metrics_test.go +++ /dev/null @@ -1,101 +0,0 @@ -package apiserver - -import ( - "context" - "fmt" - "net/url" - "testing" - "time" - - "github.com/jarcoal/httpmock" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/crowdsecurity/go-cs-lib/pkg/version" - - "github.com/crowdsecurity/crowdsec/pkg/apiclient" -) - -func TestAPICSendMetrics(t *testing.T) { - tests := []struct { - name string - duration time.Duration - expectedCalls int - setUp func(*apic) - metricsInterval time.Duration - }{ - { - name: "basic", - duration: time.Millisecond * 30, - metricsInterval: time.Millisecond * 5, - expectedCalls: 5, - setUp: func(api *apic) {}, - }, - { - name: "with some metrics", - duration: time.Millisecond * 30, - metricsInterval: time.Millisecond * 5, - expectedCalls: 5, - setUp: func(api *apic) { - api.dbClient.Ent.Machine.Delete().ExecX(context.Background()) - api.dbClient.Ent.Machine.Create(). - SetMachineId("1234"). - SetPassword(testPassword.String()). - SetIpAddress("1.2.3.4"). - SetScenarios("crowdsecurity/test"). - SetLastPush(time.Time{}). - SetUpdatedAt(time.Time{}). - ExecX(context.Background()) - - api.dbClient.Ent.Bouncer.Delete().ExecX(context.Background()) - api.dbClient.Ent.Bouncer.Create(). - SetIPAddress("1.2.3.6"). - SetName("someBouncer"). - SetAPIKey("foobar"). - SetRevoked(false). - SetLastPull(time.Time{}). - ExecX(context.Background()) - }, - }, - } - - httpmock.RegisterResponder("POST", "http://api.crowdsec.net/api/metrics/", httpmock.NewBytesResponder(200, []byte{})) - httpmock.Activate() - defer httpmock.Deactivate() - - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - url, err := url.ParseRequestURI("http://api.crowdsec.net/") - require.NoError(t, err) - - apiClient, err := apiclient.NewDefaultClient( - url, - "/api", - fmt.Sprintf("crowdsec/%s", version.String()), - nil, - ) - require.NoError(t, err) - - api := getAPIC(t) - api.pushInterval = time.Millisecond - api.pushIntervalFirst = time.Millisecond - api.apiClient = apiClient - api.metricsInterval = tc.metricsInterval - api.metricsIntervalFirst = tc.metricsInterval - tc.setUp(api) - - stop := make(chan bool) - httpmock.ZeroCallCounters() - go api.SendMetrics(stop) - time.Sleep(tc.duration) - stop <- true - - info := httpmock.GetCallCountInfo() - noResponderCalls := info["NO_RESPONDER"] - responderCalls := info["POST http://api.crowdsec.net/api/metrics/"] - assert.LessOrEqual(t, absDiff(tc.expectedCalls, responderCalls), 2) - assert.Zero(t, noResponderCalls) - }) - } -} diff --git a/pkg/apiserver/apic_test.go b/pkg/apiserver/apic_test.go index 8aeb092cd..65ca29991 100644 --- a/pkg/apiserver/apic_test.go +++ b/pkg/apiserver/apic_test.go @@ -1057,6 +1057,90 @@ func TestAPICPush(t *testing.T) { } } +func TestAPICSendMetrics(t *testing.T) { + tests := []struct { + name string + duration time.Duration + expectedCalls int + setUp func(*apic) + metricsInterval time.Duration + }{ + { + name: "basic", + duration: time.Millisecond * 30, + metricsInterval: time.Millisecond * 5, + expectedCalls: 5, + setUp: func(api *apic) {}, + }, + { + name: "with some metrics", + duration: time.Millisecond * 30, + metricsInterval: time.Millisecond * 5, + expectedCalls: 5, + setUp: func(api *apic) { + api.dbClient.Ent.Machine.Delete().ExecX(context.Background()) + api.dbClient.Ent.Machine.Create(). + SetMachineId("1234"). + SetPassword(testPassword.String()). + SetIpAddress("1.2.3.4"). + SetScenarios("crowdsecurity/test"). + SetLastPush(time.Time{}). + SetUpdatedAt(time.Time{}). + ExecX(context.Background()) + + api.dbClient.Ent.Bouncer.Delete().ExecX(context.Background()) + api.dbClient.Ent.Bouncer.Create(). + SetIPAddress("1.2.3.6"). + SetName("someBouncer"). + SetAPIKey("foobar"). + SetRevoked(false). + SetLastPull(time.Time{}). + ExecX(context.Background()) + }, + }, + } + + httpmock.RegisterResponder("POST", "http://api.crowdsec.net/api/metrics/", httpmock.NewBytesResponder(200, []byte{})) + httpmock.Activate() + defer httpmock.Deactivate() + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + url, err := url.ParseRequestURI("http://api.crowdsec.net/") + require.NoError(t, err) + + apiClient, err := apiclient.NewDefaultClient( + url, + "/api", + fmt.Sprintf("crowdsec/%s", version.String()), + nil, + ) + require.NoError(t, err) + + api := getAPIC(t) + api.pushInterval = time.Millisecond + api.pushIntervalFirst = time.Millisecond + api.apiClient = apiClient + api.metricsInterval = tc.metricsInterval + api.metricsIntervalFirst = tc.metricsInterval + tc.setUp(api) + + stop := make(chan bool) + httpmock.ZeroCallCounters() + go api.SendMetrics(stop) + time.Sleep(tc.duration) + stop <- true + + info := httpmock.GetCallCountInfo() + noResponderCalls := info["NO_RESPONDER"] + responderCalls := info["POST http://api.crowdsec.net/api/metrics/"] + assert.LessOrEqual(t, absDiff(tc.expectedCalls, responderCalls), 2) + assert.Zero(t, noResponderCalls) + }) + } +} + func TestAPICPull(t *testing.T) { api := getAPIC(t) tests := []struct { diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 3facd0f6f..c65100573 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -9,18 +9,9 @@ import ( "net" "net/http" "os" - "path/filepath" "strings" "time" - "github.com/gin-gonic/gin" - "github.com/go-co-op/gocron" - "github.com/golang-jwt/jwt/v4" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "gopkg.in/natefinch/lumberjack.v2" - "gopkg.in/tomb.v2" - "github.com/crowdsecurity/go-cs-lib/pkg/trace" "github.com/crowdsecurity/crowdsec/pkg/apiclient" @@ -31,6 +22,13 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/fflag" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/gin-gonic/gin" + "github.com/go-co-op/gocron" + "github.com/golang-jwt/jwt/v4" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "gopkg.in/natefinch/lumberjack.v2" + "gopkg.in/tomb.v2" ) var ( @@ -118,7 +116,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) { logFile := "" if config.LogMedia == "file" { - logFile = filepath.Join(config.LogDir, "crowdsec_api.log") + logFile = fmt.Sprintf("%s/crowdsec_api.log", config.LogDir) } if log.GetLevel() < log.DebugLevel { @@ -164,7 +162,15 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) { if config.CompressLogs != nil { _compress = *config.CompressLogs } - + /*cf. https://github.com/natefinch/lumberjack/issues/82 + let's create the file beforehand w/ the right perms */ + // check if file exists + _, err := os.Stat(logFile) + // create file if not exists, purposefully ignore errors + if os.IsNotExist(err) { + file, _ := os.OpenFile(logFile, os.O_RDWR|os.O_CREATE, 0600) + file.Close() + } LogOutput := &lumberjack.Logger{ Filename: logFile, MaxSize: _maxsize, //megabytes diff --git a/pkg/apiserver/controllers/v1/controller.go b/pkg/apiserver/controllers/v1/controller.go index 60da83d7d..3820bc10f 100644 --- a/pkg/apiserver/controllers/v1/controller.go +++ b/pkg/apiserver/controllers/v1/controller.go @@ -2,15 +2,17 @@ package v1 import ( "context" - "fmt" "net" + //"github.com/crowdsecurity/crowdsec/pkg/apiserver/controllers" + middlewares "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csplugin" "github.com/crowdsecurity/crowdsec/pkg/csprofiles" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/pkg/errors" ) type Controller struct { @@ -46,7 +48,7 @@ func New(cfg *ControllerV1Config) (*Controller, error) { profiles, err := csprofiles.NewProfile(cfg.ProfilesCfg) if err != nil { - return &Controller{}, fmt.Errorf("failed to compile profiles: %w", err) + return &Controller{}, errors.Wrapf(err, "failed to compile profiles") } v1 := &Controller{ diff --git a/pkg/apiserver/middlewares/v1/api_key.go b/pkg/apiserver/middlewares/v1/api_key.go index ce1bc8eee..503f4d43d 100644 --- a/pkg/apiserver/middlewares/v1/api_key.go +++ b/pkg/apiserver/middlewares/v1/api_key.go @@ -3,7 +3,7 @@ package v1 import ( "crypto/rand" "crypto/sha512" - "encoding/base64" + "encoding/hex" "fmt" "net/http" "strings" @@ -15,11 +15,9 @@ import ( log "github.com/sirupsen/logrus" ) -const ( +var ( APIKeyHeader = "X-Api-Key" bouncerContextKey = "bouncer_info" - // max allowed by bcrypt 72 = 54 bytes in base64 - dummyAPIKeySize = 54 ) type APIKey struct { @@ -33,7 +31,7 @@ func GenerateAPIKey(n int) (string, error) { if _, err := rand.Read(bytes); err != nil { return "", err } - return base64.StdEncoding.EncodeToString(bytes), nil + return hex.EncodeToString(bytes), nil } func NewAPIKey(dbClient *database.Client) *APIKey { @@ -84,7 +82,7 @@ func (a *APIKey) MiddlewareFunc() gin.HandlerFunc { if err != nil && strings.Contains(err.Error(), "bouncer not found") { //Because we have a valid cert, automatically create the bouncer in the database if it does not exist //Set a random API key, but it will never be used - apiKey, err := GenerateAPIKey(dummyAPIKeySize) + apiKey, err := GenerateAPIKey(64) if err != nil { log.WithFields(log.Fields{ "ip": c.ClientIP(), diff --git a/pkg/apiserver/middlewares/v1/jwt.go b/pkg/apiserver/middlewares/v1/jwt.go index bbd33c544..9f69f332e 100644 --- a/pkg/apiserver/middlewares/v1/jwt.go +++ b/pkg/apiserver/middlewares/v1/jwt.go @@ -81,7 +81,7 @@ func (j *JWT) Authenticator(c *gin.Context) (interface{}, error) { //Machine was not found, let's create it log.Printf("machine %s not found, create it", machineID) //let's use an apikey as the password, doesn't matter in this case (generatePassword is only available in cscli) - pwd, err := GenerateAPIKey(dummyAPIKeySize) + pwd, err := GenerateAPIKey(64) if err != nil { log.WithFields(log.Fields{ "ip": c.ClientIP(), diff --git a/pkg/apiserver/middlewares/v1/tls_auth.go b/pkg/apiserver/middlewares/v1/tls_auth.go index 87ca896a8..a0b837a41 100644 --- a/pkg/apiserver/middlewares/v1/tls_auth.go +++ b/pkg/apiserver/middlewares/v1/tls_auth.go @@ -12,6 +12,7 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/crypto/ocsp" ) @@ -175,9 +176,9 @@ func (ta *TLSAuth) isInvalid(cert *x509.Certificate, issuer *x509.Certificate) ( } revoked, err := ta.isRevoked(cert, issuer) if err != nil { - //Fail securely, if we can't check the revocation status, let's consider the cert invalid + //Fail securely, if we can't check the revokation status, let's consider the cert invalid //We may change this in the future based on users feedback, but this seems the most sensible thing to do - return true, fmt.Errorf("could not check for client certification revocation status: %w", err) + return true, errors.Wrap(err, "could not check for client certification revokation status") } return revoked, nil @@ -230,7 +231,7 @@ func (ta *TLSAuth) ValidateCert(c *gin.Context) (bool, string, error) { revoked, err := ta.isInvalid(clientCert, c.Request.TLS.VerifiedChains[0][1]) if err != nil { ta.logger.Errorf("TLSAuth: error checking if client certificate is revoked: %s", err) - return false, "", fmt.Errorf("could not check for client certification revokation status: %w", err) + return false, "", errors.Wrap(err, "could not check for client certification revokation status") } if revoked { return false, "", fmt.Errorf("client certificate is revoked") diff --git a/pkg/csconfig/common.go b/pkg/csconfig/common.go index 9d80cd95a..6add00c05 100644 --- a/pkg/csconfig/common.go +++ b/pkg/csconfig/common.go @@ -4,6 +4,7 @@ import ( "fmt" "path/filepath" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) @@ -38,7 +39,7 @@ func (c *Config) LoadCommon() error { } *k, err = filepath.Abs(*k) if err != nil { - return fmt.Errorf("failed to get absolute path of '%s': %w", *k, err) + return errors.Wrapf(err, "failed to get absolute path of '%s'", *k) } } diff --git a/pkg/csconfig/config_paths.go b/pkg/csconfig/config_paths.go index 24ff454b7..59be93ae6 100644 --- a/pkg/csconfig/config_paths.go +++ b/pkg/csconfig/config_paths.go @@ -3,6 +3,8 @@ package csconfig import ( "fmt" "path/filepath" + + "github.com/pkg/errors" ) type ConfigurationPaths struct { @@ -48,7 +50,7 @@ func (c *Config) LoadConfigurationPaths() error { } *k, err = filepath.Abs(*k) if err != nil { - return fmt.Errorf("failed to get absolute path of '%s': %w", *k, err) + return errors.Wrapf(err, "failed to get absolute path of '%s'", *k) } } diff --git a/pkg/csconfig/profiles.go b/pkg/csconfig/profiles.go index ec70fb459..41725bcf2 100644 --- a/pkg/csconfig/profiles.go +++ b/pkg/csconfig/profiles.go @@ -2,13 +2,13 @@ package csconfig import ( "bytes" - "errors" "fmt" "io" "github.com/crowdsecurity/go-cs-lib/pkg/yamlpatch" "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/pkg/errors" "gopkg.in/yaml.v2" ) @@ -53,7 +53,7 @@ func (c *LocalApiServerCfg) LoadProfiles() error { if errors.Is(err, io.EOF) { break } - return fmt.Errorf("while decoding %s: %w", c.ProfilesPath, err) + return errors.Wrapf(err, "while decoding %s", c.ProfilesPath) } c.Profiles = append(c.Profiles, &t) } diff --git a/pkg/csplugin/broker.go b/pkg/csplugin/broker.go index b1bd54dfd..6bc3b1296 100644 --- a/pkg/csplugin/broker.go +++ b/pkg/csplugin/broker.go @@ -2,7 +2,6 @@ package csplugin import ( "context" - "errors" "fmt" "io" "os" @@ -15,6 +14,7 @@ import ( "github.com/Masterminds/sprig/v3" "github.com/google/uuid" plugin "github.com/hashicorp/go-plugin" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" @@ -83,10 +83,10 @@ func (pb *PluginBroker) Init(pluginCfg *csconfig.PluginCfg, profileConfigs []*cs pb.pluginProcConfig = pluginCfg pb.pluginsTypesToDispatch = make(map[string]struct{}) if err := pb.loadConfig(configPaths.NotificationDir); err != nil { - return fmt.Errorf("while loading plugin config: %w", err) + return errors.Wrap(err, "while loading plugin config") } if err := pb.loadPlugins(configPaths.PluginDir); err != nil { - return fmt.Errorf("while loading plugin: %w", err) + return errors.Wrap(err, "while loading plugin") } pb.watcher = PluginWatcher{} pb.watcher.Init(pb.pluginConfigByName, pb.alertsByPluginName) @@ -268,7 +268,7 @@ func (pb *PluginBroker) loadPlugins(path string) error { data = []byte(csstring.StrictExpand(string(data), os.LookupEnv)) _, err = pluginClient.Configure(context.Background(), &protobufs.Config{Config: data}) if err != nil { - return fmt.Errorf("while configuring %s: %w", pc.Name, err) + return errors.Wrapf(err, "while configuring %s", pc.Name) } log.Infof("registered plugin %s", pc.Name) pb.notificationPluginByName[pc.Name] = pluginClient @@ -354,7 +354,7 @@ func ParsePluginConfigFile(path string) ([]PluginConfig, error) { parsedConfigs := make([]PluginConfig, 0) yamlFile, err := os.Open(path) if err != nil { - return nil, fmt.Errorf("while opening %s: %w", path, err) + return parsedConfigs, errors.Wrapf(err, "while opening %s", path) } dec := yaml.NewDecoder(yamlFile) dec.SetStrict(true) @@ -365,7 +365,7 @@ func ParsePluginConfigFile(path string) ([]PluginConfig, error) { if errors.Is(err, io.EOF) { break } - return nil, fmt.Errorf("while decoding %s got error %s", path, err) + return []PluginConfig{}, fmt.Errorf("while decoding %s got error %s", path, err) } // if the yaml document is empty, skip if reflect.DeepEqual(pc, PluginConfig{}) { diff --git a/pkg/csplugin/broker_suite_test.go b/pkg/csplugin/broker_suite_test.go index 6e3e51407..4c7cdd6eb 100644 --- a/pkg/csplugin/broker_suite_test.go +++ b/pkg/csplugin/broker_suite_test.go @@ -14,6 +14,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) + type PluginSuite struct { suite.Suite @@ -22,19 +23,21 @@ type PluginSuite struct { // full path to the built plugin binary builtBinary string - runDir string // temporary directory for each test - pluginDir string // (config_paths.plugin_dir) - notifDir string // (config_paths.notification_dir) - pluginBinary string // full path to the plugin binary (unique for each test) - pluginConfig string // full path to the notification config (unique for each test) + runDir string // temporary directory for each test + pluginDir string // (config_paths.plugin_dir) + notifDir string // (config_paths.notification_dir) + pluginBinary string // full path to the plugin binary (unique for each test) + pluginConfig string // full path to the notification config (unique for each test) pluginBroker *PluginBroker } + func TestPluginSuite(t *testing.T) { suite.Run(t, new(PluginSuite)) } + func (s *PluginSuite) SetupSuite() { var err error @@ -54,12 +57,14 @@ func (s *PluginSuite) SetupSuite() { require.NoError(t, err, "while building dummy plugin") } + func (s *PluginSuite) TearDownSuite() { t := s.T() err := os.RemoveAll(s.buildDir) require.NoError(t, err) } + func copyFile(src string, dst string) error { s, err := os.Open(src) if err != nil { @@ -94,6 +99,7 @@ func (s *PluginSuite) TearDownTest() { s.TearDownSubTest() } + func (s *PluginSuite) SetupSubTest() { var err error t := s.T() @@ -119,7 +125,7 @@ func (s *PluginSuite) SetupSubTest() { require.NoError(t, err, "while copying built binary") err = os.Chmod(s.pluginBinary, 0o744) require.NoError(t, err, "chmod 0744 %s", s.pluginBinary) - + s.pluginConfig = path.Join(s.notifDir, "dummy.yaml") err = copyFile("testdata/dummy.yaml", s.pluginConfig) require.NoError(t, err, "while copying plugin config") diff --git a/pkg/csplugin/broker_test.go b/pkg/csplugin/broker_test.go index 991b89ed2..467fadf45 100644 --- a/pkg/csplugin/broker_test.go +++ b/pkg/csplugin/broker_test.go @@ -22,6 +22,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/models" ) + func (s *PluginSuite) permissionSetter(perm os.FileMode) func(*testing.T) { return func(t *testing.T) { err := os.Chmod(s.pluginBinary, perm) @@ -29,28 +30,30 @@ func (s *PluginSuite) permissionSetter(perm os.FileMode) func(*testing.T) { } } -func (s *PluginSuite) readconfig() PluginConfig { +func (s *PluginSuite) readconfig() (PluginConfig) { var config PluginConfig t := s.T() orig, err := os.ReadFile(s.pluginConfig) - require.NoError(t, err, "unable to read config file %s", s.pluginConfig) + require.NoError(t, err,"unable to read config file %s", s.pluginConfig) err = yaml.Unmarshal(orig, &config) - require.NoError(t, err, "unable to unmarshal config file") - + require.NoError(t, err,"unable to unmarshal config file") + return config } + func (s *PluginSuite) writeconfig(config PluginConfig) { t := s.T() data, err := yaml.Marshal(&config) - require.NoError(t, err, "unable to marshal config file") + require.NoError(t, err,"unable to marshal config file") err = os.WriteFile(s.pluginConfig, data, 0644) - require.NoError(t, err, "unable to write config file %s", s.pluginConfig) + require.NoError(t, err,"unable to write config file %s", s.pluginConfig) } + func (s *PluginSuite) TestBrokerInit() { tests := []struct { name string @@ -59,7 +62,7 @@ func (s *PluginSuite) TestBrokerInit() { expectedErr string }{ { - name: "valid config", + name: "valid config", }, { name: "group writable binary", @@ -346,7 +349,7 @@ func (s *PluginSuite) TestBrokerRunSimple() { DefaultEmptyTicker = 50 * time.Millisecond t := s.T() - + pb, err := s.InitBroker(nil) assert.NoError(t, err) diff --git a/pkg/csplugin/broker_win_test.go b/pkg/csplugin/broker_win_test.go index 01262b1fd..3d7498c0d 100644 --- a/pkg/csplugin/broker_win_test.go +++ b/pkg/csplugin/broker_win_test.go @@ -33,7 +33,7 @@ func (s *PluginSuite) TestBrokerInit() { expectedErr string }{ { - name: "valid config", + name: "valid config", }, { name: "no plugin dir", diff --git a/pkg/csplugin/listfiles.go b/pkg/csplugin/listfiles.go index c91be03b4..2dea44f4f 100644 --- a/pkg/csplugin/listfiles.go +++ b/pkg/csplugin/listfiles.go @@ -13,9 +13,10 @@ func listFilesAtPath(path string) ([]string, error) { return nil, err } for _, file := range files { - if !file.IsDir() { + if ! file.IsDir() { filePaths = append(filePaths, filepath.Join(path, file.Name())) } } return filePaths, nil } + diff --git a/pkg/csplugin/listfiles_test.go b/pkg/csplugin/listfiles_test.go index 09102ef0d..8bcedaa1f 100644 --- a/pkg/csplugin/listfiles_test.go +++ b/pkg/csplugin/listfiles_test.go @@ -27,9 +27,9 @@ func TestListFilesAtPath(t *testing.T) { require.NoError(t, err) tests := []struct { - name string - path string - want []string + name string + path string + want []string expectedErr string }{ { @@ -41,8 +41,8 @@ func TestListFilesAtPath(t *testing.T) { }, }, { - name: "invalid directory", - path: "./foo/bar/", + name: "invalid directory", + path: "./foo/bar/", expectedErr: "open ./foo/bar/: " + cstest.PathNotFoundMessage, }, } diff --git a/pkg/csplugin/notifier.go b/pkg/csplugin/notifier.go index 8ab1aa923..64a1e6e71 100644 --- a/pkg/csplugin/notifier.go +++ b/pkg/csplugin/notifier.go @@ -4,10 +4,9 @@ import ( "context" "fmt" + "github.com/crowdsecurity/crowdsec/pkg/protobufs" plugin "github.com/hashicorp/go-plugin" "google.golang.org/grpc" - - "github.com/crowdsecurity/crowdsec/pkg/protobufs" ) type Notifier interface { diff --git a/pkg/csplugin/utils.go b/pkg/csplugin/utils.go index 5e19dee38..67707c829 100644 --- a/pkg/csplugin/utils.go +++ b/pkg/csplugin/utils.go @@ -3,7 +3,6 @@ package csplugin import ( - "errors" "fmt" "io/fs" "math" @@ -14,6 +13,8 @@ import ( "strconv" "strings" "syscall" + + "github.com/pkg/errors" ) func CheckCredential(uid int, gid int) *syscall.SysProcAttr { @@ -34,7 +35,7 @@ func (pb *PluginBroker) CreateCmd(binaryPath string) (*exec.Cmd, error) { } cmd.SysProcAttr, err = getProcessAttr(pb.pluginProcConfig.User, pb.pluginProcConfig.Group) if err != nil { - return nil, fmt.Errorf("while getting process attributes: %w", err) + return nil, errors.Wrap(err, "while getting process attributes") } cmd.SysProcAttr.Credential.NoSetGroups = true } @@ -104,17 +105,17 @@ func pluginIsValid(path string) error { // check if it exists if details, err = os.Stat(path); err != nil { - return fmt.Errorf("plugin at %s does not exist: %w", path, err) + return errors.Wrap(err, fmt.Sprintf("plugin at %s does not exist", path)) } // check if it is owned by current user currentUser, err := user.Current() if err != nil { - return fmt.Errorf("while getting current user: %w", err) + return errors.Wrap(err, "while getting current user") } currentUID, err := getUID(currentUser.Username) if err != nil { - return fmt.Errorf("while looking up the current uid: %w", err) + return errors.Wrap(err, "while looking up the current uid") } stat := details.Sys().(*syscall.Stat_t) if stat.Uid != currentUID { diff --git a/pkg/csplugin/utils_windows.go b/pkg/csplugin/utils_windows.go index dfb11aff5..874e30021 100644 --- a/pkg/csplugin/utils_windows.go +++ b/pkg/csplugin/utils_windows.go @@ -13,6 +13,7 @@ import ( "syscall" "unsafe" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/sys/windows" ) @@ -53,38 +54,38 @@ func CheckPerms(path string) error { systemSid, err := windows.CreateWellKnownSid(windows.WELL_KNOWN_SID_TYPE(windows.WinLocalSystemSid)) if err != nil { - return fmt.Errorf("while creating SYSTEM well known sid: %w", err) + return errors.Wrap(err, "while creating SYSTEM well known sid") } adminSid, err := windows.CreateWellKnownSid(windows.WELL_KNOWN_SID_TYPE(windows.WinBuiltinAdministratorsSid)) if err != nil { - return fmt.Errorf("while creating built-in Administrators well known sid: %w", err) + return errors.Wrap(err, "while creating built-in Administrators well known sid") } currentUser, err := user.Current() if err != nil { - return fmt.Errorf("while getting current user: %w", err) + return errors.Wrap(err, "while getting current user") } currentUserSid, _, _, err := windows.LookupSID("", currentUser.Username) if err != nil { - return fmt.Errorf("while looking up current user sid: %w", err) + return errors.Wrap(err, "while looking up current user sid") } sd, err := windows.GetNamedSecurityInfo(path, windows.SE_FILE_OBJECT, windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION) if err != nil { - return fmt.Errorf("while getting owner security info: %w", err) + return errors.Wrap(err, "while getting owner security info") } if !sd.IsValid() { - return fmt.Errorf("security descriptor is invalid") + return errors.New("security descriptor is invalid") } owner, _, err := sd.Owner() if err != nil { - return fmt.Errorf("while getting owner: %w", err) + return errors.Wrap(err, "while getting owner") } if !owner.IsValid() { - return fmt.Errorf("owner is invalid") + return errors.New("owner is invalid") } if !owner.Equals(systemSid) && !owner.Equals(currentUserSid) && !owner.Equals(adminSid) { @@ -93,7 +94,7 @@ func CheckPerms(path string) error { dacl, _, err := sd.DACL() if err != nil { - return fmt.Errorf("while getting DACL: %w", err) + return errors.Wrap(err, "while getting DACL") } if dacl == nil { @@ -101,7 +102,7 @@ func CheckPerms(path string) error { } if err != nil { - return fmt.Errorf("while looking up current user sid: %w", err) + return errors.Wrap(err, "while looking up current user sid") } rs := reflect.ValueOf(dacl).Elem() @@ -123,7 +124,7 @@ func CheckPerms(path string) error { ace := &AccessAllowedAce{} ret, _, _ := procGetAce.Call(uintptr(unsafe.Pointer(dacl)), uintptr(i), uintptr(unsafe.Pointer(&ace))) if ret == 0 { - return fmt.Errorf("while getting ACE: %w", windows.GetLastError()) + return errors.Wrap(windows.GetLastError(), "while getting ACE") } log.Debugf("ACE %d: %+v\n", i, ace) @@ -161,14 +162,14 @@ func getProcessAtr() (*syscall.SysProcAttr, error) { err := windows.OpenProcessToken(proc, windows.TOKEN_DUPLICATE|windows.TOKEN_ADJUST_DEFAULT| windows.TOKEN_QUERY|windows.TOKEN_ASSIGN_PRIMARY|windows.TOKEN_ADJUST_GROUPS|windows.TOKEN_ADJUST_PRIVILEGES, &procToken) if err != nil { - return nil, fmt.Errorf("while opening process token: %w", err) + return nil, errors.Wrapf(err, "while opening process token") } defer procToken.Close() err = windows.DuplicateTokenEx(procToken, 0, nil, windows.SecurityImpersonation, windows.TokenPrimary, &token) if err != nil { - return nil, fmt.Errorf("while duplicating token: %w", err) + return nil, errors.Wrapf(err, "while duplicating token") } //Remove all privileges from the token @@ -176,7 +177,7 @@ func getProcessAtr() (*syscall.SysProcAttr, error) { err = windows.AdjustTokenPrivileges(token, true, nil, 0, nil, nil) if err != nil { - return nil, fmt.Errorf("while adjusting token privileges: %w", err) + return nil, errors.Wrapf(err, "while adjusting token privileges") } //Run the plugin as a medium integrity level process @@ -194,7 +195,7 @@ func getProcessAtr() (*syscall.SysProcAttr, error) { (*byte)(unsafe.Pointer(tml)), tml.Size()) if err != nil { token.Close() - return nil, fmt.Errorf("while setting token information: %w", err) + return nil, errors.Wrapf(err, "while setting token information") } return &windows.SysProcAttr{ @@ -208,7 +209,7 @@ func (pb *PluginBroker) CreateCmd(binaryPath string) (*exec.Cmd, error) { cmd := exec.Command(binaryPath) cmd.SysProcAttr, err = getProcessAtr() if err != nil { - return nil, fmt.Errorf("while getting process attributes: %w", err) + return nil, errors.Wrap(err, "while getting process attributes") } return cmd, err } @@ -228,7 +229,7 @@ func pluginIsValid(path string) error { // check if it exists if _, err = os.Stat(path); err != nil { - return fmt.Errorf("plugin at %s does not exist", path) + return errors.Wrap(err, fmt.Sprintf("plugin at %s does not exist", path)) } // check if it is owned by root diff --git a/pkg/csplugin/watcher.go b/pkg/csplugin/watcher.go index bec0302e4..983a53c89 100644 --- a/pkg/csplugin/watcher.go +++ b/pkg/csplugin/watcher.go @@ -4,10 +4,9 @@ import ( "sync" "time" + "github.com/crowdsecurity/crowdsec/pkg/models" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" - - "github.com/crowdsecurity/crowdsec/pkg/models" ) /* diff --git a/pkg/csplugin/watcher_test.go b/pkg/csplugin/watcher_test.go index 391a94810..94d8d0617 100644 --- a/pkg/csplugin/watcher_test.go +++ b/pkg/csplugin/watcher_test.go @@ -7,12 +7,9 @@ import ( "testing" "time" - "github.com/stretchr/testify/require" - "gopkg.in/tomb.v2" - - "github.com/crowdsecurity/go-cs-lib/pkg/cstest" - "github.com/crowdsecurity/crowdsec/pkg/models" + "gopkg.in/tomb.v2" + "gotest.tools/v3/assert" ) var ctx = context.Background() @@ -67,7 +64,7 @@ func TestPluginWatcherInterval(t *testing.T) { ct, cancel := context.WithTimeout(ctx, time.Microsecond) defer cancel() err := listenChannelWithTimeout(ct, pw.PluginEvents) - cstest.RequireErrorContains(t, err, "context deadline exceeded") + assert.ErrorContains(t, err, "context deadline exceeded") resetTestTomb(&testTomb, &pw) testTomb = tomb.Tomb{} pw.Start(&testTomb) @@ -75,7 +72,7 @@ func TestPluginWatcherInterval(t *testing.T) { ct, cancel = context.WithTimeout(ctx, time.Millisecond*5) defer cancel() err = listenChannelWithTimeout(ct, pw.PluginEvents) - require.NoError(t, err) + assert.NilError(t, err) resetTestTomb(&testTomb, &pw) // This is to avoid the int complaining } @@ -99,7 +96,7 @@ func TestPluginAlertCountWatcher(t *testing.T) { ct, cancel := context.WithTimeout(ctx, time.Second) defer cancel() err := listenChannelWithTimeout(ct, pw.PluginEvents) - cstest.RequireErrorContains(t, err, "context deadline exceeded") + assert.ErrorContains(t, err, "context deadline exceeded") // Channel won't contain any events since threshold is not crossed. resetWatcherAlertCounter(&pw) @@ -107,7 +104,7 @@ func TestPluginAlertCountWatcher(t *testing.T) { ct, cancel = context.WithTimeout(ctx, time.Second) defer cancel() err = listenChannelWithTimeout(ct, pw.PluginEvents) - cstest.RequireErrorContains(t, err, "context deadline exceeded") + assert.ErrorContains(t, err, "context deadline exceeded") // Channel will contain an event since threshold is crossed. resetWatcherAlertCounter(&pw) @@ -115,6 +112,6 @@ func TestPluginAlertCountWatcher(t *testing.T) { ct, cancel = context.WithTimeout(ctx, time.Second) defer cancel() err = listenChannelWithTimeout(ct, pw.PluginEvents) - require.NoError(t, err) + assert.NilError(t, err) resetTestTomb(&testTomb, &pw) } diff --git a/pkg/csprofiles/csprofiles.go b/pkg/csprofiles/csprofiles.go index 70dea3d7a..29e6cdf36 100644 --- a/pkg/csprofiles/csprofiles.go +++ b/pkg/csprofiles/csprofiles.go @@ -6,13 +6,12 @@ import ( "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) type Runtime struct { @@ -48,10 +47,10 @@ func NewProfile(profilesCfg []*csconfig.ProfileCfg) ([]*Runtime, error) { runtime.DebugFilters = make([]*exprhelpers.ExprDebugger, len(profile.Filters)) runtime.Cfg = profile if runtime.Cfg.OnSuccess != "" && runtime.Cfg.OnSuccess != "continue" && runtime.Cfg.OnSuccess != "break" { - return []*Runtime{}, fmt.Errorf("invalid 'on_success' for '%s': %s", profile.Name, runtime.Cfg.OnSuccess) + return []*Runtime{}, errors.Wrapf(err, "invalid 'on_success' for '%s' : %s", profile.Name, runtime.Cfg.OnSuccess) } if runtime.Cfg.OnFailure != "" && runtime.Cfg.OnFailure != "continue" && runtime.Cfg.OnFailure != "break" && runtime.Cfg.OnFailure != "apply" { - return []*Runtime{}, fmt.Errorf("invalid 'on_failure' for '%s' : %s", profile.Name, runtime.Cfg.OnFailure) + return []*Runtime{}, errors.Wrapf(err, "invalid 'on_failure' for '%s' : %s", profile.Name, runtime.Cfg.OnFailure) } for fIdx, filter := range profile.Filters { diff --git a/pkg/csprofiles/csprofiles_test.go b/pkg/csprofiles/csprofiles_test.go index 8adf68291..81b21e1fb 100644 --- a/pkg/csprofiles/csprofiles_test.go +++ b/pkg/csprofiles/csprofiles_test.go @@ -5,11 +5,10 @@ import ( "reflect" "testing" - "github.com/stretchr/testify/require" - "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/models" + "gotest.tools/v3/assert" ) var ( @@ -96,7 +95,7 @@ func TestNewProfile(t *testing.T) { } profile, _ := NewProfile(profilesCfg) fmt.Printf("expected : %+v | result : %+v", test.expectedNbProfile, len(profile)) - require.Len(t, profile, test.expectedNbProfile) + assert.Equal(t, test.expectedNbProfile, len(profile)) }) } } @@ -200,7 +199,7 @@ func TestEvaluateProfile(t *testing.T) { t.Errorf("EvaluateProfile() got1 = %v, want %v", got1, tt.expectedMatchStatus) } if tt.expectedDuration != "" { - require.Equal(t, tt.expectedDuration, *got[0].Duration, "The two durations should be the same") + assert.Equal(t, tt.expectedDuration, *got[0].Duration, "The two durations should be the same") } }) } diff --git a/pkg/cticlient/example/fire.go b/pkg/cticlient/example/fire.go index e52922571..7bcf814a0 100644 --- a/pkg/cticlient/example/fire.go +++ b/pkg/cticlient/example/fire.go @@ -44,9 +44,6 @@ func main() { } for _, item := range items { - if item.State == "refused" { - continue - } banDuration := time.Until(item.Expiration.Time) allItems = append(allItems, []string{ item.Ip, diff --git a/pkg/cticlient/types.go b/pkg/cticlient/types.go index 2ad0a6eb3..1d6550d00 100644 --- a/pkg/cticlient/types.go +++ b/pkg/cticlient/types.go @@ -120,7 +120,7 @@ type FireItem struct { BackgroundNoiseScore *int `json:"background_noise_score"` Scores CTIScores `json:"scores"` References []CTIReferences `json:"references"` - State string `json:"state"` + Status string `json:"status"` Expiration CustomTime `json:"expiration"` } diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index b8a09d4c1..0e508d9f1 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -148,7 +148,7 @@ func GetItemByPath(itemType string, itemPath string) (*Item, error) { finalName := "" f, err := os.Lstat(itemPath) if err != nil { - return nil, fmt.Errorf("while performing lstat on %s: %w", itemPath, err) + return nil, errors.Wrapf(err, "while performing lstat on %s", itemPath) } if f.Mode()&os.ModeSymlink == 0 { @@ -158,7 +158,7 @@ func GetItemByPath(itemType string, itemPath string) (*Item, error) { /*resolve the symlink to hub file*/ pathInHub, err := os.Readlink(itemPath) if err != nil { - return nil, fmt.Errorf("while reading symlink of %s: %w", itemPath, err) + return nil, errors.Wrapf(err, "while reading symlink of %s", itemPath) } //extract author from path fname := filepath.Base(pathInHub) @@ -240,7 +240,7 @@ func GetInstalledScenariosAsString() ([]string, error) { items, err := GetInstalledScenarios() if err != nil { - return nil, fmt.Errorf("while fetching scenarios: %w", err) + return nil, errors.Wrap(err, "while fetching scenarios") } for _, it := range items { retStr = append(retStr, it.Name) @@ -281,7 +281,7 @@ func GetInstalledParsersAsString() ([]string, error) { items, err := GetInstalledParsers() if err != nil { - return nil, fmt.Errorf("while fetching parsers: %w", err) + return nil, errors.Wrap(err, "while fetching parsers") } for _, it := range items { retStr = append(retStr, it.Name) @@ -308,7 +308,7 @@ func GetInstalledPostOverflowsAsString() ([]string, error) { items, err := GetInstalledPostOverflows() if err != nil { - return nil, fmt.Errorf("while fetching post overflows: %w", err) + return nil, errors.Wrap(err, "while fetching post overflows") } for _, it := range items { retStr = append(retStr, it.Name) @@ -321,9 +321,8 @@ func GetInstalledCollectionsAsString() ([]string, error) { items, err := GetInstalledCollections() if err != nil { - return nil, fmt.Errorf("while fetching collections: %w", err) + return nil, errors.Wrap(err, "while fetching collections") } - for _, it := range items { retStr = append(retStr, it.Name) } diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index f91b0dced..b0e300aa7 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -25,7 +25,7 @@ import ( var responseByPath map[string]string func TestItemStatus(t *testing.T) { - cfg := envSetup(t) + cfg := envSetup() defer envTearDown(cfg) err := UpdateHubIdx(cfg.Hub) @@ -73,7 +73,7 @@ func TestItemStatus(t *testing.T) { } func TestGetters(t *testing.T) { - cfg := envSetup(t) + cfg := envSetup() defer envTearDown(cfg) err := UpdateHubIdx(cfg.Hub) @@ -134,7 +134,7 @@ func TestGetters(t *testing.T) { } func TestIndexDownload(t *testing.T) { - cfg := envSetup(t) + cfg := envSetup() defer envTearDown(cfg) err := UpdateHubIdx(cfg.Hub) @@ -155,16 +155,10 @@ func getTestCfg() (cfg *csconfig.Config) { return } -func envSetup(t *testing.T) *csconfig.Config { +func envSetup() *csconfig.Config { resetResponseByPath() log.SetLevel(log.DebugLevel) cfg := getTestCfg() - - defaultTransport := http.DefaultClient.Transport - t.Cleanup(func() { - http.DefaultClient.Transport = defaultTransport - }) - //Mock the http client http.DefaultClient.Transport = newMockTransport() @@ -327,7 +321,7 @@ func TestInstallParser(t *testing.T) { - check its status - remove it */ - cfg := envSetup(t) + cfg := envSetup() defer envTearDown(cfg) getHubIdxOrFail(t) @@ -359,7 +353,7 @@ func TestInstallCollection(t *testing.T) { - check its status - remove it */ - cfg := envSetup(t) + cfg := envSetup() defer envTearDown(cfg) getHubIdxOrFail(t) diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go index 17b5c0e00..6923c6d05 100644 --- a/pkg/cwhub/download.go +++ b/pkg/cwhub/download.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" @@ -255,7 +256,7 @@ func downloadData(dataFolder string, force bool, reader io.Reader) error { dec := yaml.NewDecoder(reader) for { - data := &DataSet{} + data := &types.DataSet{} err = dec.Decode(data) if err != nil { if errors.Is(err, io.EOF) { @@ -271,7 +272,7 @@ func downloadData(dataFolder string, force bool, reader io.Reader) error { } } if download || force { - err = GetData(data.Data, dataFolder) + err = types.GetData(data.Data, dataFolder) if err != nil { return errors.Wrap(err, "while getting data") } diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index 4133e2272..af1e938d7 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -4,12 +4,12 @@ import ( "fmt" "path/filepath" - "github.com/enescakir/emoji" - log "github.com/sirupsen/logrus" - "golang.org/x/mod/semver" - "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwversion" + "github.com/enescakir/emoji" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "golang.org/x/mod/semver" ) // pick a hub branch corresponding to the current crowdsec version. @@ -79,11 +79,11 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo item, err := DownloadLatest(csConfig.Hub, item, force, true) if err != nil { - return fmt.Errorf("while downloading %s: %w", item.Name, err) + return errors.Wrapf(err, "while downloading %s", item.Name) } if err := AddItem(obtype, item); err != nil { - return fmt.Errorf("while adding %s: %w", item.Name, err) + return errors.Wrapf(err, "while adding %s", item.Name) } if downloadOnly { @@ -93,11 +93,11 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo item, err = EnableItem(csConfig.Hub, item) if err != nil { - return fmt.Errorf("while enabling %s: %w", item.Name, err) + return errors.Wrapf(err, "while enabling %s", item.Name) } if err := AddItem(obtype, item); err != nil { - return fmt.Errorf("while adding %s: %w", item.Name, err) + return errors.Wrapf(err, "while adding %s", item.Name) } log.Infof("Enabled %s", item.Name) diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index bf6e84fb3..143967002 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -9,7 +9,7 @@ import ( //Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection // We expect the new scenario to be installed func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { - cfg := envSetup(t) + cfg := envSetup() defer envTearDown(cfg) // fresh install of collection @@ -55,7 +55,7 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { // Install a collection, disable a scenario. // Upgrade should install should not enable/download the disabled scenario. func TestUpgradeConfigInDisabledSceanarioShouldNotBeInstalled(t *testing.T) { - cfg := envSetup(t) + cfg := envSetup() defer envTearDown(cfg) // fresh install of collection @@ -103,7 +103,7 @@ func getHubIdxOrFail(t *testing.T) { // Upgrade should not enable/download the disabled scenario. // Upgrade should install and enable the newly added scenario. func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) { - cfg := envSetup(t) + cfg := envSetup() defer envTearDown(cfg) // fresh install of collection diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/loader.go index ecb5d0a3a..961e9a02e 100644 --- a/pkg/cwhub/loader.go +++ b/pkg/cwhub/loader.go @@ -2,17 +2,16 @@ package cwhub import ( "encoding/json" - "errors" "fmt" "os" "path/filepath" "sort" "strings" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/mod/semver" - - "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) /*the walk/parser_visit function can't receive extra args*/ @@ -369,7 +368,7 @@ func GetHubIdx(hub *csconfig.Hub) error { log.Debugf("loading hub idx %s", hub.HubIndexFile) bidx, err := os.ReadFile(hub.HubIndexFile) if err != nil { - return fmt.Errorf("unable to read index file: %w", err) + return errors.Wrap(err, "unable to read index file") } ret, err := LoadPkgIndex(bidx) if err != nil { diff --git a/pkg/database/alerts.go b/pkg/database/alerts.go index 688288cee..ad0117236 100644 --- a/pkg/database/alerts.go +++ b/pkg/database/alerts.go @@ -9,10 +9,6 @@ import ( "strings" "time" - "github.com/davecgh/go-spew/spew" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/alert" @@ -24,6 +20,9 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/database/ent/predicate" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/davecgh/go-spew/spew" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) const ( @@ -202,7 +201,7 @@ func (c *Client) CreateOrUpdateAlert(machineID string, alertItem *models.Alert) if strings.ToLower(*decisionItem.Scope) == "ip" || strings.ToLower(*decisionItem.Scope) == "range" { sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(*decisionItem.Value) if err != nil { - return "", errors.Wrapf(InvalidIPOrRange, "invalid addr/range %s : %s", *decisionItem.Value, err) + return "", errors.Wrapf(ParseDurationFail, "invalid addr/range %s : %s", *decisionItem.Value, err) } } decisionDuration, err := time.ParseDuration(*decisionItem.Duration) @@ -392,7 +391,7 @@ func (c *Client) UpdateCommunityBlocklist(alertItem *models.Alert) (int, int, in if rollbackErr != nil { log.Errorf("rollback error: %s", rollbackErr) } - return 0, 0, 0, errors.Wrapf(InvalidIPOrRange, "invalid addr/range %s : %s", *decisionItem.Value, err) + return 0, 0, 0, errors.Wrapf(ParseDurationFail, "invalid addr/range %s : %s", *decisionItem.Value, err) } } /*bulk insert some new decisions*/ @@ -565,7 +564,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ } marshallMetas, err := json.Marshal(eventItem.Meta) if err != nil { - return nil, errors.Wrapf(MarshalFail, "event meta '%v' : %s", eventItem.Meta, err) + return []string{}, errors.Wrapf(MarshalFail, "event meta '%v' : %s", eventItem.Meta, err) } //the serialized field is too big, let's try to progressively strip it @@ -583,7 +582,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ marshallMetas, err = json.Marshal(eventItem.Meta) if err != nil { - return nil, errors.Wrapf(MarshalFail, "event meta '%v' : %s", eventItem.Meta, err) + return []string{}, errors.Wrapf(MarshalFail, "event meta '%v' : %s", eventItem.Meta, err) } if event.SerializedValidator(string(marshallMetas)) == nil { valid = true @@ -612,7 +611,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ } events, err = c.Ent.Event.CreateBulk(eventBulk...).Save(c.CTX) if err != nil { - return nil, errors.Wrapf(BulkError, "creating alert events: %s", err) + return []string{}, errors.Wrapf(BulkError, "creating alert events: %s", err) } } @@ -625,7 +624,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ } metas, err = c.Ent.Meta.CreateBulk(metaBulk...).Save(c.CTX) if err != nil { - return nil, errors.Wrapf(BulkError, "creating alert meta: %s", err) + return []string{}, errors.Wrapf(BulkError, "creating alert meta: %s", err) } } @@ -638,14 +637,14 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ duration, err := time.ParseDuration(*decisionItem.Duration) if err != nil { - return nil, errors.Wrapf(ParseDurationFail, "decision duration '%+v' : %s", *decisionItem.Duration, err) + return []string{}, errors.Wrapf(ParseDurationFail, "decision duration '%+v' : %s", *decisionItem.Duration, err) } /*if the scope is IP or Range, convert the value to integers */ if strings.ToLower(*decisionItem.Scope) == "ip" || strings.ToLower(*decisionItem.Scope) == "range" { sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(*decisionItem.Value) if err != nil { - return nil, fmt.Errorf("%s: %w", *decisionItem.Value, InvalidIPOrRange) + return []string{}, errors.Wrapf(ParseDurationFail, "invalid addr/range %s : %s", *decisionItem.Value, err) } } @@ -668,7 +667,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ if len(decisionBulk) == decisionBulkSize { decisionsCreateRet, err := c.Ent.Decision.CreateBulk(decisionBulk...).Save(c.CTX) if err != nil { - return nil, errors.Wrapf(BulkError, "creating alert decisions: %s", err) + return []string{}, errors.Wrapf(BulkError, "creating alert decisions: %s", err) } decisions = append(decisions, decisionsCreateRet...) @@ -681,7 +680,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ } decisionsCreateRet, err := c.Ent.Decision.CreateBulk(decisionBulk...).Save(c.CTX) if err != nil { - return nil, errors.Wrapf(BulkError, "creating alert decisions: %s", err) + return []string{}, errors.Wrapf(BulkError, "creating alert decisions: %s", err) } decisions = append(decisions, decisionsCreateRet...) } @@ -720,7 +719,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ if len(bulk) == bulkSize { alerts, err := c.Ent.Alert.CreateBulk(bulk...).Save(c.CTX) if err != nil { - return nil, errors.Wrapf(BulkError, "bulk creating alert : %s", err) + return []string{}, errors.Wrapf(BulkError, "bulk creating alert : %s", err) } for alertIndex, a := range alerts { ret = append(ret, strconv.Itoa(a.ID)) @@ -729,7 +728,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ for _, d2 := range decisionsChunk { _, err := c.Ent.Alert.Update().Where(alert.IDEQ(a.ID)).AddDecisions(d2...).Save(c.CTX) if err != nil { - return nil, fmt.Errorf("error while updating decisions: %s", err) + return []string{}, fmt.Errorf("error while updating decisions: %s", err) } } } @@ -745,7 +744,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ alerts, err := c.Ent.Alert.CreateBulk(bulk...).Save(c.CTX) if err != nil { - return nil, errors.Wrapf(BulkError, "leftovers creating alert : %s", err) + return []string{}, errors.Wrapf(BulkError, "leftovers creating alert : %s", err) } for alertIndex, a := range alerts { @@ -755,7 +754,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ for _, d2 := range decisionsChunk { _, err := c.Ent.Alert.Update().Where(alert.IDEQ(a.ID)).AddDecisions(d2...).Save(c.CTX) if err != nil { - return nil, fmt.Errorf("error while updating decisions: %s", err) + return []string{}, fmt.Errorf("error while updating decisions: %s", err) } } } diff --git a/pkg/database/bouncers.go b/pkg/database/bouncers.go index 98bfd4587..4cd32d839 100644 --- a/pkg/database/bouncers.go +++ b/pkg/database/bouncers.go @@ -4,10 +4,9 @@ import ( "fmt" "time" - "github.com/pkg/errors" - "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/bouncer" + "github.com/pkg/errors" ) func (c *Client) SelectBouncer(apiKeyHash string) (*ent.Bouncer, error) { diff --git a/pkg/database/config.go b/pkg/database/config.go index 8c3578ad5..90a85490f 100644 --- a/pkg/database/config.go +++ b/pkg/database/config.go @@ -1,10 +1,9 @@ package database import ( - "github.com/pkg/errors" - "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/configitem" + "github.com/pkg/errors" ) func (c *Client) GetConfigItem(key string) (*string, error) { diff --git a/pkg/database/decisions.go b/pkg/database/decisions.go index 5dacc23d4..61483f9fd 100644 --- a/pkg/database/decisions.go +++ b/pkg/database/decisions.go @@ -2,17 +2,17 @@ package database import ( "fmt" - "strconv" "strings" "time" - "entgo.io/ent/dialect/sql" - "github.com/pkg/errors" + "strconv" + "entgo.io/ent/dialect/sql" "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/decision" "github.com/crowdsecurity/crowdsec/pkg/database/ent/predicate" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/pkg/errors" ) type DecisionsByScenario struct { diff --git a/pkg/database/machines.go b/pkg/database/machines.go index 7a010fbfb..48243324d 100644 --- a/pkg/database/machines.go +++ b/pkg/database/machines.go @@ -5,12 +5,12 @@ import ( "time" "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - "golang.org/x/crypto/bcrypt" "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/machine" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/pkg/errors" + "golang.org/x/crypto/bcrypt" ) const CapiMachineID = types.CAPIOrigin diff --git a/pkg/exprhelpers/jsonextract.go b/pkg/exprhelpers/jsonextract.go index a616588a7..a874122ff 100644 --- a/pkg/exprhelpers/jsonextract.go +++ b/pkg/exprhelpers/jsonextract.go @@ -175,8 +175,8 @@ func UnmarshalJSON(params ...any) (any, error) { err := json.Unmarshal([]byte(jsonBlob), &out) if err != nil { log.Errorf("UnmarshalJSON : %s", err) - return nil, err + return "", err } target[key] = out - return nil, nil + return "", nil } diff --git a/pkg/hubtest/hubtest.go b/pkg/hubtest/hubtest.go index c1aa4251c..36415f746 100644 --- a/pkg/hubtest/hubtest.go +++ b/pkg/hubtest/hubtest.go @@ -7,6 +7,7 @@ import ( "path/filepath" "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/pkg/errors" ) type HubTest struct { @@ -104,7 +105,7 @@ func (h *HubTest) LoadAllTests() error { for _, f := range testsFolder { if f.IsDir() { if _, err := h.LoadTestItem(f.Name()); err != nil { - return fmt.Errorf("while loading %s: %w", f.Name(), err) + return errors.Wrapf(err, "while loading %s", f.Name()) } } } diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index 1ec7c5f44..c3e842bbb 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -10,6 +10,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/parser" + "github.com/crowdsecurity/crowdsec/pkg/types" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" ) @@ -22,7 +23,7 @@ type HubTestItemConfig struct { LogType string `yaml:"log_type"` Labels map[string]string `yaml:"labels"` IgnoreParsers bool `yaml:"ignore_parsers"` // if we test a scenario, we don't want to assert on Parser - OverrideStatics []parser.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00 + OverrideStatics []types.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00 } type HubIndex struct { diff --git a/pkg/hubtest/parser_assert.go b/pkg/hubtest/parser_assert.go index 95400b50d..3d52f37e5 100644 --- a/pkg/hubtest/parser_assert.go +++ b/pkg/hubtest/parser_assert.go @@ -12,14 +12,14 @@ import ( "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/enescakir/emoji" "github.com/fatih/color" + "github.com/pkg/errors" diff "github.com/r3labs/diff/v2" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" - - "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" - "github.com/crowdsecurity/crowdsec/pkg/types" ) type AssertFail struct { @@ -164,7 +164,7 @@ func (p *ParserAssert) RunExpression(expression string) (interface{}, error) { if err != nil { log.Warningf("running : %s", expression) log.Warningf("runtime error : %s", err) - return output, fmt.Errorf("while running expression %s: %w", expression, err) + return output, errors.Wrapf(err, "while running expression %s", expression) } return output, nil } diff --git a/pkg/hubtest/scenario_assert.go b/pkg/hubtest/scenario_assert.go index 2e2a4e9c8..d9ec4dddc 100644 --- a/pkg/hubtest/scenario_assert.go +++ b/pkg/hubtest/scenario_assert.go @@ -11,11 +11,11 @@ import ( "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" - log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" - "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" ) type ScenarioAssert struct { @@ -149,7 +149,7 @@ func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) { env := map[string]interface{}{"results": *s.TestData} if runtimeFilter, err = expr.Compile(expression, exprhelpers.GetExprOptions(env)...); err != nil { - return nil, err + return output, err } // if debugFilter, err = exprhelpers.NewDebugger(assert, expr.Env(env)); err != nil { // log.Warningf("Failed building debugher for %s : %s", assert, err) @@ -162,7 +162,7 @@ func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) { if err != nil { log.Warningf("running : %s", expression) log.Warningf("runtime error : %s", err) - return nil, fmt.Errorf("while running expression %s: %w", expression, err) + return output, errors.Wrapf(err, "while running expression %s", expression) } return output, nil } diff --git a/pkg/leakybucket/README.md b/pkg/leakybucket/README.md index 4614eddba..5254f33b2 100644 --- a/pkg/leakybucket/README.md +++ b/pkg/leakybucket/README.md @@ -2,102 +2,86 @@ ## Bucket concepts -The Leakybucket is used for decision making. Under certain conditions, -enriched events are poured into these buckets. When these buckets are +Leakybucket is used for decision making. Under certain conditions +enriched events are poured in these buckets. When these buckets are full, we raise a new event. After this event is raised the bucket is destroyed. There are many types of buckets, and we welcome any new useful design of buckets. -Usually, the bucket configuration generates the creation of many -buckets. They are differentiated by a field called stackkey. When two -events arrive with the same stackkey they go in the same matching +Usually the bucket configuration generates the creation of many +buckets. They are differenciated by a field called stackkey. When two +events arrives with the same stackkey they go in the same matching bucket. The very purpose of these buckets is to detect clients that exceed a -certain rate of attempts to do something (ssh connection, http -authentication failure, etc...). Thus, the most used stackkey field is +certain rate of attemps to do something (ssh connection, http +authentication failure, etc...). Thus, the most use stackkey field is often the source_ip. ## Standard leaky buckets Default buckets have two main configuration options: - * capacity: number of events the bucket can hold. When the capacity is reached and a new event is poured, a new event is raised. We call this type of event overflow. This is an int. - * leakspeed: duration needed for an event to leak. When an event - leaks, it disappears from the bucket. + leaks, it disappear from the bucket. ## Trigger -A Trigger is a special type of bucket with a capacity of zero. Thus, when an -event is poured into a trigger, it always raises an overflow. +It's a special type of bucket with a zero capacity. Thus, when an +event is poured in a trigger, it always raises an overflow. ## Uniq -A Uniq is a bucket working like the standard leaky bucket except for one +It's a bucket working as the standard leaky bucket except for one thing: a filter returns a property for each event and only one occurrence of this property is allowed in the bucket, thus the bucket is called uniq. ## Counter -A Counter is a special type of bucket with an infinite capacity and an -infinite leakspeed (it never overflows, nor leaks). Nevertheless, +It's a special type of bucket with an infinite capacity and an +infinite leakspeed (it never overflows, neither leaks). Nevertheless, the event is raised after a fixed duration. The option is called duration. -## Bayesian - -A Bayesian is a special bucket that runs bayesian inference instead of -counting events. Each event must have its likelihoods specified in the -yaml file under `prob_given_benign` and `prob_given_evil`. The bucket -will continue evaluating events until the posterior goes above the -threshold (triggering the overflow) or the duration (specified by leakspeed) -expires. - ## Available configuration options for buckets ### Fields for standard buckets * type: mandatory field. Must be one of "leaky", "trigger", "uniq" or "counter" - -* name: mandatory field, but the value is totally open. Nevertheless, +* name: mandatory field, but the value is totally open. Nevertheless this value will tag the events raised by the bucket. - -* filter: mandatory field. It's a filter that is run to decide whether - an event matches the bucket or not. The filter has to return +* filter: mandatory field. It's a filter that is run when the decision + to make an event match the bucket or not. The filter have to return a boolean. As a filter implementation we use https://github.com/antonmedv/expr - * capacity: [mandatory for now, shouldn't be mandatory in the final version] it's the size of the bucket. When pouring in a bucket already with size events, it overflows. - -* leakspeed: leakspeed is a time duration (it has to be parsed by - https://golang.org/pkg/time/#ParseDuration). After each interval, an +* leakspeed: leakspeed is a time duration (has to be parseable by + https://golang.org/pkg/time/#ParseDuration). After each interval an event is leaked from the bucket. - * stackkey: mandatory field. This field is used to differentiate on - which instance of the bucket the matching events will be poured. - When an unknown stackkey is seen in an event, a new bucket is created. - -* on_overflow: optional field, that tells what to do when the - bucket is returning the overflow event. As of today, the possibilities - are "ban,1h", "Reprocess" or "Delete". - Reprocess is used to send the raised event back to the event pool to - be matched against buckets + which bucket ongoing events will be poured. When an unknown stackkey + is seen in an event a new bucket is created. +* on_overflow: optional field, that tells the what to do when the + bucket is returning the overflow event. As of today, the possibility + are these: "ban,1h", "Reprocess", "Delete". + Reprocess is used to send the raised event back in the event pool to + be matched agains buckets ### Fields for special buckets #### Uniq - * uniq_filter: an expression that must comply with the syntax defined - in https://github.com/antonmedv/expr and must return a string. - All strings returned by this filter in the same buckets have to be different. - Thus if a string is seen twice, the event is dismissed. +Uniq has an extra field uniq_filter which is too use the filter +implementation from https://github.com/antonmedv/expr. The filter must +return a string. All strins returned by this filter in the same +buckets have to be different. Thus, if a string is seen twice it is +dismissed. #### Trigger @@ -105,27 +89,11 @@ Capacity and leakspeed are not relevant for this kind of bucket. #### Counter - * duration: the Counter will be destroyed after this interval - has elapsed since its creation. The duration must be parsed - by https://golang.org/pkg/time/#ParseDuration. - Nevertheless, this kind of bucket is often used with an infinite - leakspeed and an infinite capacity [capacity set to -1 for now]. - -#### Bayesian - - * bayesian_prior: The prior to start with - * bayesian_threshold: The threshold for the posterior to trigger the overflow. - * bayesian_conditions: List of Bayesian conditions with likelihoods - -Bayesian Conditions are built from: - * condition: The expr for this specific condition to be true - * prob_given_evil: The likelihood an IP satisfies the condition given the fact - that it is a maliscious IP - * prob_given_benign: The likelihood an IP satisfies the condition given the fact - that it is a benign IP - * guillotine: Bool to stop the condition from getting evaluated if it has - evaluated to true once. This should be used if evaluating the condition is - computationally expensive. +It's a special kind of bucket that raise an event and is destroyed +after a fixed duration. The configuration field used is duration and +must be parseable by https://golang.org/pkg/time/#ParseDuration. +Nevertheless, this kind of bucket is often used with an infinite +leakspeed and an infinite capacity [capacity set to -1 for now]. ## Add examples here @@ -158,17 +126,17 @@ Bayesian Conditions are built from: [This is not dry enough to have many details here, but:] -The bucket code is triggered by runPour in pour.go, by calling the `leaky.PourItemToHolders` function. -There is one struct called buckets which is for now a +The bucket code is triggered by `InfiniBucketify` in main.go. +There's one struct called buckets which is for now a `map[string]interface{}` that holds all buckets. The key of this map -is derived from the filter configured for the bucket and its -stackkey. This looks complicated, but it allows us to use -only one struct. This is done in buckets.go. +is derivated from the filter configured for the bucket and its +stackkey. This looks like complicated, but in fact it allows us to use +only one structs. This is done in buckets.go. -On top of that the implementation defines only the standard leaky -bucket. A goroutine is launched for every bucket (`bucket.go`). This +On top of that the implementation define only the standard leaky +bucket. A goroutine is launched for every buckets (bucket.go). This goroutine manages the life of the bucket. For special buckets, hooks are defined at initialization time in -manager.go. Hooks are called when relevant by the bucket goroutine -when events are poured and/or when a bucket overflows. +manager.go. Hooks are called when relevant by the bucket gorourine +when events are poured and/or when bucket overflows. \ No newline at end of file diff --git a/pkg/leakybucket/bayesian.go b/pkg/leakybucket/bayesian.go deleted file mode 100644 index bd9aaed96..000000000 --- a/pkg/leakybucket/bayesian.go +++ /dev/null @@ -1,163 +0,0 @@ -package leakybucket - -import ( - "fmt" - - "github.com/antonmedv/expr" - "github.com/antonmedv/expr/vm" - "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" - "github.com/crowdsecurity/crowdsec/pkg/types" -) - -type RawBayesianCondition struct { - ConditionalFilterName string `yaml:"condition"` - ProbGivenEvil float32 `yaml:"prob_given_evil"` - ProbGivenBenign float32 `yaml:"prob_given_benign"` - Guillotine bool `yaml:"guillotine,omitempty"` -} - -type BayesianEvent struct { - rawCondition RawBayesianCondition - conditionalFilterRuntime *vm.Program - guillotineState bool -} - -type BayesianBucket struct { - bayesianEventArray []*BayesianEvent - prior float32 - threshold float32 - posterior float32 - DumbProcessor -} - -func updateProbability(prior, probGivenEvil, ProbGivenBenign float32) float32 { - numerator := probGivenEvil * prior - denominator := numerator + ProbGivenBenign*(1-prior) - - return numerator / denominator -} - -func (c *BayesianBucket) OnBucketInit(g *BucketFactory) error { - var err error - BayesianEventArray := make([]*BayesianEvent, len(g.BayesianConditions)) - - if conditionalExprCache == nil { - conditionalExprCache = make(map[string]vm.Program) - } - conditionalExprCacheLock.Lock() - - for index, bcond := range g.BayesianConditions { - var bayesianEvent BayesianEvent - bayesianEvent.rawCondition = bcond - err = bayesianEvent.compileCondition() - if err != nil { - return err - } - BayesianEventArray[index] = &bayesianEvent - } - conditionalExprCacheLock.Unlock() - c.bayesianEventArray = BayesianEventArray - - c.prior = g.BayesianPrior - c.threshold = g.BayesianThreshold - - return err -} - -func (c *BayesianBucket) AfterBucketPour(b *BucketFactory) func(types.Event, *Leaky) *types.Event { - return func(msg types.Event, l *Leaky) *types.Event { - c.posterior = c.prior - l.logger.Debugf("starting bayesian evaluation with prior: %v", c.posterior) - - for _, bevent := range c.bayesianEventArray { - err := bevent.bayesianUpdate(c, msg, l) - if err != nil { - l.logger.Errorf("bayesian update failed for %s with %s", bevent.rawCondition.ConditionalFilterName, err) - } - } - - l.logger.Debugf("value of posterior after events : %v", c.posterior) - - if c.posterior > c.threshold { - l.logger.Debugf("Bayesian bucket overflow") - l.Ovflw_ts = l.Last_ts - l.Out <- l.Queue - return nil - } - - return &msg - } -} - -func (b *BayesianEvent) bayesianUpdate(c *BayesianBucket, msg types.Event, l *Leaky) error { - var condition, ok bool - - if b.conditionalFilterRuntime == nil { - l.logger.Tracef("empty conditional filter runtime for %s", b.rawCondition.ConditionalFilterName) - return nil - } - - l.logger.Tracef("guillotine value for %s : %v", b.rawCondition.ConditionalFilterName, b.getGuillotineState()) - if b.getGuillotineState() { - l.logger.Tracef("guillotine already triggered for %s", b.rawCondition.ConditionalFilterName) - l.logger.Tracef("condition true updating prior for: %s", b.rawCondition.ConditionalFilterName) - c.posterior = updateProbability(c.posterior, b.rawCondition.ProbGivenEvil, b.rawCondition.ProbGivenBenign) - l.logger.Tracef("new value of posterior : %v", c.posterior) - return nil - } - - l.logger.Debugf("running condition expression: %s", b.rawCondition.ConditionalFilterName) - ret, err := expr.Run(b.conditionalFilterRuntime, map[string]interface{}{"evt": &msg, "queue": l.Queue, "leaky": l}) - if err != nil { - return fmt.Errorf("unable to run conditional filter: %s", err) - } - - l.logger.Tracef("bayesian bucket expression %s returned : %v", b.rawCondition.ConditionalFilterName, ret) - if condition, ok = ret.(bool); !ok { - return fmt.Errorf("bayesian condition unexpected non-bool return: %T", ret) - } - - l.logger.Tracef("condition %T updating prior for: %s", condition, b.rawCondition.ConditionalFilterName) - if condition { - c.posterior = updateProbability(c.posterior, b.rawCondition.ProbGivenEvil, b.rawCondition.ProbGivenBenign) - b.triggerGuillotine() - } else { - c.posterior = updateProbability(c.posterior, 1-b.rawCondition.ProbGivenEvil, 1-b.rawCondition.ProbGivenBenign) - } - l.logger.Tracef("new value of posterior: %v", c.posterior) - - return nil -} - -func (b *BayesianEvent) getGuillotineState() bool { - if b.rawCondition.Guillotine { - return b.guillotineState - } - return false -} - -func (b *BayesianEvent) triggerGuillotine() { - b.guillotineState = true -} - -func (b *BayesianEvent) compileCondition() error { - var err error - var compiledExpr *vm.Program - - if compiled, ok := conditionalExprCache[b.rawCondition.ConditionalFilterName]; ok { - b.conditionalFilterRuntime = &compiled - return nil - } - - conditionalExprCacheLock.Unlock() - //release the lock during compile same as coditional bucket - compiledExpr, err = expr.Compile(b.rawCondition.ConditionalFilterName, exprhelpers.GetExprOptions(map[string]interface{}{"queue": &Queue{}, "leaky": &Leaky{}, "evt": &types.Event{}})...) - if err != nil { - return fmt.Errorf("bayesian condition compile error: %w", err) - } - b.conditionalFilterRuntime = compiledExpr - conditionalExprCacheLock.Lock() - conditionalExprCache[b.rawCondition.ConditionalFilterName] = *compiledExpr - - return nil -} diff --git a/pkg/leakybucket/bucket.go b/pkg/leakybucket/bucket.go index 286c51f11..004d5b9d8 100644 --- a/pkg/leakybucket/bucket.go +++ b/pkg/leakybucket/bucket.go @@ -191,10 +191,6 @@ func FromFactory(bucketFactory BucketFactory) *Leaky { l.conditionalOverflow = true l.Duration = l.BucketConfig.leakspeed } - - if l.BucketConfig.Type == "bayesian" { - l.Duration = l.BucketConfig.leakspeed - } return l } diff --git a/pkg/leakybucket/manager_load.go b/pkg/leakybucket/manager_load.go index dc1f4ed51..1e212f815 100644 --- a/pkg/leakybucket/manager_load.go +++ b/pkg/leakybucket/manager_load.go @@ -51,9 +51,6 @@ type BucketFactory struct { Profiling bool `yaml:"profiling"` //Profiling, if true, will make the bucket record pours/overflows/etc. OverflowFilter string `yaml:"overflow_filter"` //OverflowFilter if present, is a filter that must return true for the overflow to go through ConditionalOverflow string `yaml:"condition"` //condition if present, is an expression that must return true for the bucket to overflow - BayesianPrior float32 `yaml:"bayesian_prior"` - BayesianThreshold float32 `yaml:"bayesian_threshold"` - BayesianConditions []RawBayesianCondition `yaml:"bayesian_conditions"` //conditions for the bayesian bucket ScopeType types.ScopeType `yaml:"scope,omitempty"` //to enforce a different remediation than blocking an IP. Will default this to IP BucketName string `yaml:"-"` Filename string `yaml:"-"` @@ -123,25 +120,6 @@ func ValidateFactory(bucketFactory *BucketFactory) error { if bucketFactory.leakspeed == 0 { return fmt.Errorf("bad leakspeed for conditional bucket '%s'", bucketFactory.LeakSpeed) } - } else if bucketFactory.Type == "bayesian" { - if bucketFactory.BayesianConditions == nil { - return fmt.Errorf("bayesian bucket must have bayesian conditions") - } - if bucketFactory.BayesianPrior == 0 { - return fmt.Errorf("bayesian bucket must have a valid, non-zero prior") - } - if bucketFactory.BayesianThreshold == 0 { - return fmt.Errorf("bayesian bucket must have a valid, non-zero threshold") - } - if bucketFactory.BayesianPrior > 1 { - return fmt.Errorf("bayesian bucket must have a valid, non-zero prior") - } - if bucketFactory.BayesianThreshold > 1 { - return fmt.Errorf("bayesian bucket must have a valid, non-zero threshold") - } - if bucketFactory.Capacity != -1 { - return fmt.Errorf("bayesian bucket must have capacity -1") - } } else { return fmt.Errorf("unknown bucket type '%s'", bucketFactory.Type) } @@ -338,8 +316,6 @@ func LoadBucket(bucketFactory *BucketFactory, tomb *tomb.Tomb) error { bucketFactory.processors = append(bucketFactory.processors, &DumbProcessor{}) case "conditional": bucketFactory.processors = append(bucketFactory.processors, &DumbProcessor{}) - case "bayesian": - bucketFactory.processors = append(bucketFactory.processors, &DumbProcessor{}) default: return fmt.Errorf("invalid type '%s' in %s : %v", bucketFactory.Type, bucketFactory.Filename, err) } @@ -379,11 +355,6 @@ func LoadBucket(bucketFactory *BucketFactory, tomb *tomb.Tomb) error { bucketFactory.processors = append(bucketFactory.processors, &ConditionalOverflow{}) } - if bucketFactory.BayesianThreshold != 0 { - bucketFactory.logger.Tracef("Adding bayesian processor") - bucketFactory.processors = append(bucketFactory.processors, &BayesianBucket{}) - } - if len(bucketFactory.Data) > 0 { for _, data := range bucketFactory.Data { if data.DestPath == "" { diff --git a/pkg/leakybucket/manager_load_test.go b/pkg/leakybucket/manager_load_test.go index 513f11ff3..bb3df75cd 100644 --- a/pkg/leakybucket/manager_load_test.go +++ b/pkg/leakybucket/manager_load_test.go @@ -119,25 +119,3 @@ func TestCounterBucketsConfig(t *testing.T) { } } - -func TestBayesianBucketsConfig(t *testing.T) { - var CfgTests = []cfgTest{ - - //basic valid counter - {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: -1, Filter: "true", BayesianPrior: 0.5, BayesianThreshold: 0.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, true, true}, - //bad capacity - {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: 1, Filter: "true", BayesianPrior: 0.5, BayesianThreshold: 0.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, false, false}, - //missing prior - {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: -1, Filter: "true", BayesianThreshold: 0.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, false, false}, - //missing threshold - {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: -1, Filter: "true", BayesianPrior: 0.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, false, false}, - //bad prior - {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: -1, Filter: "true", BayesianPrior: 1.5, BayesianThreshold: 0.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, false, false}, - //bad threshold - {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: -1, Filter: "true", BayesianPrior: 0.5, BayesianThreshold: 1.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, false, false}, - } - if err := runTest(CfgTests); err != nil { - t.Fatalf("%s", err) - } - -} diff --git a/pkg/leakybucket/overflows.go b/pkg/leakybucket/overflows.go index 45446b8f6..d6131cd26 100644 --- a/pkg/leakybucket/overflows.go +++ b/pkg/leakybucket/overflows.go @@ -6,14 +6,15 @@ import ( "sort" "strconv" - "github.com/antonmedv/expr" - "github.com/davecgh/go-spew/spew" - "github.com/go-openapi/strfmt" - log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/crowdsec/pkg/alertcontext" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/davecgh/go-spew/spew" + "github.com/go-openapi/strfmt" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + + "github.com/antonmedv/expr" ) // SourceFromEvent extracts and formats a valid models.Source object from an Event @@ -52,7 +53,7 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e if leaky.scopeType.RunTimeFilter != nil { retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, map[string]interface{}{"evt": &evt}) if err != nil { - return srcs, fmt.Errorf("while running scope filter: %w", err) + return srcs, errors.Wrapf(err, "while running scope filter") } value, ok := retValue.(string) if !ok { @@ -127,7 +128,7 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e if leaky.scopeType.RunTimeFilter != nil { retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, map[string]interface{}{"evt": &evt}) if err != nil { - return srcs, fmt.Errorf("while running scope filter: %w", err) + return srcs, errors.Wrapf(err, "while running scope filter") } value, ok := retValue.(string) @@ -144,7 +145,7 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e } retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, map[string]interface{}{"evt": &evt}) if err != nil { - return srcs, fmt.Errorf("while running scope filter: %w", err) + return srcs, errors.Wrapf(err, "while running scope filter") } value, ok := retValue.(string) @@ -216,7 +217,7 @@ func alertFormatSource(leaky *Leaky, queue *Queue) (map[string]models.Source, st for _, evt := range queue.Queue { srcs, err := SourceFromEvent(evt, leaky) if err != nil { - return nil, "", fmt.Errorf("while extracting scope from bucket %s: %w", leaky.Name, err) + return nil, "", errors.Wrapf(err, "while extracting scope from bucket %s", leaky.Name) } for key, src := range srcs { if source_type == types.Undefined { @@ -275,7 +276,7 @@ func NewAlert(leaky *Leaky, queue *Queue) (types.RuntimeAlert, error) { //Get the sources from Leaky/Queue sources, source_scope, err := alertFormatSource(leaky, queue) if err != nil { - return runtimeAlert, fmt.Errorf("unable to collect sources from bucket: %w", err) + return runtimeAlert, errors.Wrap(err, "unable to collect sources from bucket") } runtimeAlert.Sources = sources //Include source info in format string diff --git a/pkg/leakybucket/tests/guillotine-bayesian-bucket/bucket.yaml b/pkg/leakybucket/tests/guillotine-bayesian-bucket/bucket.yaml deleted file mode 100644 index 8e8c26e6f..000000000 --- a/pkg/leakybucket/tests/guillotine-bayesian-bucket/bucket.yaml +++ /dev/null @@ -1,21 +0,0 @@ -type: bayesian -name: test/guillotine-bayesian -debug: true -description: "bayesian bucket" -filter: "evt.Meta.log_type == 'http_access-log' || evt.Meta.log_type == 'ssh_access-log'" -groupby: evt.Meta.source_ip -bayesian_prior: 0.5 -bayesian_threshold: 0.8 -bayesian_conditions: -- condition: evt.Meta.http_path == "/" - prob_given_evil: 0.8 - prob_given_benign: 0.2 - guillotine : true -- condition: evt.Meta.ssh_user == "admin" - prob_given_evil: 0.9 - prob_given_benign: 0.5 - guillotine : true -leakspeed: 30s -capacity: -1 -labels: - type: overflow_1 \ No newline at end of file diff --git a/pkg/leakybucket/tests/guillotine-bayesian-bucket/scenarios.yaml b/pkg/leakybucket/tests/guillotine-bayesian-bucket/scenarios.yaml deleted file mode 100644 index 05e1557cf..000000000 --- a/pkg/leakybucket/tests/guillotine-bayesian-bucket/scenarios.yaml +++ /dev/null @@ -1 +0,0 @@ - - filename: {{.TestDirectory}}/bucket.yaml \ No newline at end of file diff --git a/pkg/leakybucket/tests/guillotine-bayesian-bucket/test.json b/pkg/leakybucket/tests/guillotine-bayesian-bucket/test.json deleted file mode 100644 index 07b7b6a6e..000000000 --- a/pkg/leakybucket/tests/guillotine-bayesian-bucket/test.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "lines": [ - { - "Line": { - "Labels": { - "type": "nginx" - }, - "Raw": "don't care" - }, - "MarshaledTime": "2020-01-01T10:00:00.000Z", - "Meta": { - "source_ip": "2a00:1450:4007:816::200e", - "log_type": "http_access-log", - "http_path": "/" - } - }, - { - "Line": { - "Labels": { - "type": "nginx" - }, - "Raw": "don't care" - }, - "MarshaledTime": "2020-01-01T10:00:00.000Z", - "Meta": { - "source_ip": "2a00:1450:4007:816::200e", - "log_type": "ssh_access-log", - "ssh_user": "admin" - } - } - ], - "results": [ - { - "Type" : 1, - "Alert": { - "sources" : { - "2a00:1450:4007:816::200e": { - "ip": "2a00:1450:4007:816::200e", - "scope": "Ip", - "value": "2a00:1450:4007:816::200e" - } - }, - "Alert" : { - "scenario": "test/guillotine-bayesian", - "events_count": 2 - } - } - } - ] - } \ No newline at end of file diff --git a/pkg/leakybucket/tests/multiple-bayesian-bucket/bucket.yaml b/pkg/leakybucket/tests/multiple-bayesian-bucket/bucket.yaml deleted file mode 100644 index 1110fb7b8..000000000 --- a/pkg/leakybucket/tests/multiple-bayesian-bucket/bucket.yaml +++ /dev/null @@ -1,21 +0,0 @@ -type: bayesian -name: test/multiple-bayesian -debug: true -description: "bayesian bucket" -filter: "evt.Meta.log_type == 'http_access-log' || evt.Meta.log_type == 'ssh_access-log'" -groupby: evt.Meta.source_ip -bayesian_prior: 0.5 -bayesian_threshold: 0.8 -bayesian_conditions: -- condition: evt.Meta.http_path == "/" - prob_given_evil: 0.8 - prob_given_benign: 0.2 - guillotine : true -- condition: evt.Meta.ssh_user == "admin" - prob_given_evil: 0.9 - prob_given_benign: 0.5 - guillotine : true -leakspeed: 30s -capacity: -1 -labels: - type: overflow_1 \ No newline at end of file diff --git a/pkg/leakybucket/tests/multiple-bayesian-bucket/scenarios.yaml b/pkg/leakybucket/tests/multiple-bayesian-bucket/scenarios.yaml deleted file mode 100644 index 05e1557cf..000000000 --- a/pkg/leakybucket/tests/multiple-bayesian-bucket/scenarios.yaml +++ /dev/null @@ -1 +0,0 @@ - - filename: {{.TestDirectory}}/bucket.yaml \ No newline at end of file diff --git a/pkg/leakybucket/tests/multiple-bayesian-bucket/test.json b/pkg/leakybucket/tests/multiple-bayesian-bucket/test.json deleted file mode 100644 index 69454a6ed..000000000 --- a/pkg/leakybucket/tests/multiple-bayesian-bucket/test.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "lines": [ - { - "Line": { - "Labels": { - "type": "nginx" - }, - "Raw": "don't care" - }, - "MarshaledTime": "2020-01-01T10:00:00.000Z", - "Meta": { - "source_ip": "2a00:1450:4007:816::200e", - "log_type": "http_access-log", - "http_path": "/" - } - }, - { - "Line": { - "Labels": { - "type": "nginx" - }, - "Raw": "don't care" - }, - "MarshaledTime": "2020-01-01T10:00:00.000Z", - "Meta": { - "source_ip": "1.2.3.4", - "log_type": "ssh_access-log", - "ssh_user": "admin" - } - }, - { - "Line": { - "Labels": { - "type": "nginx" - }, - "Raw": "don't care" - }, - "MarshaledTime": "2020-01-01T10:00:00.000Z", - "Meta": { - "source_ip": "2a00:1450:4007:816::200e", - "log_type": "ssh_access-log", - "ssh_user": "admin" - } - } - ], - "results": [ - { - "Type" : 1, - "Alert": { - "sources" : { - "2a00:1450:4007:816::200e": { - "ip": "2a00:1450:4007:816::200e", - "scope": "Ip", - "value": "2a00:1450:4007:816::200e" - } - }, - "Alert" : { - "scenario": "test/multiple-bayesian", - "events_count": 2 - } - } - } - ] - } \ No newline at end of file diff --git a/pkg/leakybucket/tests/simple-bayesian-bucket/bucket.yaml b/pkg/leakybucket/tests/simple-bayesian-bucket/bucket.yaml deleted file mode 100644 index 21a4ab074..000000000 --- a/pkg/leakybucket/tests/simple-bayesian-bucket/bucket.yaml +++ /dev/null @@ -1,19 +0,0 @@ -type: bayesian -name: test/simple-bayesian -debug: true -description: "bayesian bucket" -filter: "evt.Meta.log_type == 'http_access-log' || evt.Meta.log_type == 'ssh_access-log'" -groupby: evt.Meta.source_ip -bayesian_prior: 0.5 -bayesian_threshold: 0.8 -bayesian_conditions: -- condition: any(queue.Queue, {.Meta.http_path == "/"}) - prob_given_evil: 0.8 - prob_given_benign: 0.2 -- condition: any(queue.Queue, {.Meta.ssh_user == "admin"}) - prob_given_evil: 0.9 - prob_given_benign: 0.5 -leakspeed: 30s -capacity: -1 -labels: - type: overflow_1 \ No newline at end of file diff --git a/pkg/leakybucket/tests/simple-bayesian-bucket/scenarios.yaml b/pkg/leakybucket/tests/simple-bayesian-bucket/scenarios.yaml deleted file mode 100644 index 05e1557cf..000000000 --- a/pkg/leakybucket/tests/simple-bayesian-bucket/scenarios.yaml +++ /dev/null @@ -1 +0,0 @@ - - filename: {{.TestDirectory}}/bucket.yaml \ No newline at end of file diff --git a/pkg/leakybucket/tests/simple-bayesian-bucket/test.json b/pkg/leakybucket/tests/simple-bayesian-bucket/test.json deleted file mode 100644 index a5807c4b6..000000000 --- a/pkg/leakybucket/tests/simple-bayesian-bucket/test.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "lines": [ - { - "Line": { - "Labels": { - "type": "nginx" - }, - "Raw": "don't care" - }, - "MarshaledTime": "2020-01-01T10:00:00.000Z", - "Meta": { - "source_ip": "2a00:1450:4007:816::200e", - "log_type": "http_access-log", - "http_path": "/" - } - }, - { - "Line": { - "Labels": { - "type": "nginx" - }, - "Raw": "don't care" - }, - "MarshaledTime": "2020-01-01T10:00:00.000Z", - "Meta": { - "source_ip": "2a00:1450:4007:816::200e", - "log_type": "ssh_access-log", - "ssh_user": "admin" - } - } - ], - "results": [ - { - "Type" : 1, - "Alert": { - "sources" : { - "2a00:1450:4007:816::200e": { - "ip": "2a00:1450:4007:816::200e", - "scope": "Ip", - "value": "2a00:1450:4007:816::200e" - } - }, - "Alert" : { - "scenario": "test/simple-bayesian", - "events_count": 2 - } - } - } - ] - } \ No newline at end of file diff --git a/pkg/metabase/api.go b/pkg/metabase/api.go index 7235ff7f1..99cbf9ec7 100644 --- a/pkg/metabase/api.go +++ b/pkg/metabase/api.go @@ -12,7 +12,7 @@ import ( log "github.com/sirupsen/logrus" ) -type MBClient struct { +type APIClient struct { CTX *sling.Sling Client *http.Client } @@ -35,15 +35,15 @@ var ( } ) -func NewMBClient(url string) (*MBClient, error) { +func NewAPIClient(url string) (*APIClient, error) { httpClient := &http.Client{Timeout: 20 * time.Second} - return &MBClient{ + return &APIClient{ CTX: sling.New().Client(httpClient).Base(url).Set("User-Agent", fmt.Sprintf("crowdsec/%s", version.String())), Client: httpClient, }, nil } -func (h *MBClient) Do(method string, route string, body interface{}) (interface{}, interface{}, error) { +func (h *APIClient) Do(method string, route string, body interface{}) (interface{}, interface{}, error) { var Success interface{} var Error interface{} var resp *http.Response @@ -80,6 +80,6 @@ func (h *MBClient) Do(method string, route string, body interface{}) (interface{ } // Set set headers as key:value -func (h *MBClient) Set(key string, value string) { +func (h *APIClient) Set(key string, value string) { h.CTX = h.CTX.Set(key, value) } diff --git a/pkg/metabase/database.go b/pkg/metabase/database.go index 273d06dee..0a7890fac 100644 --- a/pkg/metabase/database.go +++ b/pkg/metabase/database.go @@ -7,13 +7,14 @@ import ( "strings" "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/pkg/errors" ) type Database struct { DBUrl string Model *Model Config *csconfig.DatabaseCfg - Client *MBClient + Client *APIClient Details *Details // in case mysql host is 127.0.0.1 the ip address of mysql/pgsql host will be the docker gateway since metabase run in a container } @@ -40,7 +41,7 @@ type Model struct { Schedules map[string]interface{} `json:"schedules"` } -func NewDatabase(config *csconfig.DatabaseCfg, client *MBClient, remoteDBAddr string) (*Database, error) { +func NewDatabase(config *csconfig.DatabaseCfg, client *APIClient, remoteDBAddr string) (*Database, error) { var details *Details database := Database{} @@ -79,13 +80,13 @@ func (d *Database) Update() error { data, err := json.Marshal(success) if err != nil { - return fmt.Errorf("update sqlite db response (marshal): %w", err) + return errors.Wrap(err, "update sqlite db response (marshal)") } model := Model{} if err := json.Unmarshal(data, &model); err != nil { - return fmt.Errorf("update sqlite db response (unmarshal): %w", err) + return errors.Wrap(err, "update sqlite db response (unmarshal)") } model.Details = d.Details _, errormsg, err = d.Client.Do("PUT", routes[databaseEndpoint], model) diff --git a/pkg/metabase/metabase.go b/pkg/metabase/metabase.go index cdbe65ec8..c12d9166f 100644 --- a/pkg/metabase/metabase.go +++ b/pkg/metabase/metabase.go @@ -4,7 +4,6 @@ import ( "archive/zip" "bytes" "context" - "errors" "fmt" "io" "net/http" @@ -16,14 +15,15 @@ import ( "github.com/docker/docker/client" log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/pkg/errors" + "gopkg.in/yaml.v2" ) type Metabase struct { Config *Config - Client *MBClient + Client *APIClient Container *Container Database *Database InternalDBURL string @@ -80,7 +80,7 @@ func (m *Metabase) Init(containerName string) error { return fmt.Errorf("database '%s' not supported", m.Config.Database.Type) } - m.Client, err = NewMBClient(m.Config.ListenURL) + m.Client, err = NewAPIClient(m.Config.ListenURL) if err != nil { return err } @@ -90,7 +90,7 @@ func (m *Metabase) Init(containerName string) error { } m.Container, err = NewContainer(m.Config.ListenAddr, m.Config.ListenPort, m.Config.DBPath, containerName, metabaseImage, DBConnectionURI, m.Config.DockerGroupID) if err != nil { - return fmt.Errorf("container init: %w", err) + return errors.Wrap(err, "container init") } return nil @@ -151,36 +151,36 @@ func SetupMetabase(dbConfig *csconfig.DatabaseCfg, listenAddr string, listenPort }, } if err := metabase.Init(containerName); err != nil { - return nil, fmt.Errorf("metabase setup init: %w", err) + return nil, errors.Wrap(err, "metabase setup init") } if err := metabase.DownloadDatabase(false); err != nil { - return nil, fmt.Errorf("metabase db download: %w", err) + return nil, errors.Wrap(err, "metabase db download") } if err := metabase.Container.Create(); err != nil { - return nil, fmt.Errorf("container create: %w", err) + return nil, errors.Wrap(err, "container create") } if err := metabase.Container.Start(); err != nil { - return nil, fmt.Errorf("container start: %w", err) + return nil, errors.Wrap(err, "container start") } log.Infof("waiting for metabase to be up (can take up to a minute)") if err := metabase.WaitAlive(); err != nil { - return nil, fmt.Errorf("wait alive: %w", err) + return nil, errors.Wrap(err, "wait alive") } if err := metabase.Database.Update(); err != nil { - return nil, fmt.Errorf("update database: %w", err) + return nil, errors.Wrap(err, "update database") } if err := metabase.Scan(); err != nil { - return nil, fmt.Errorf("db scan: %w", err) + return nil, errors.Wrap(err, "db scan") } if err := metabase.ResetCredentials(); err != nil { - return nil, fmt.Errorf("reset creds: %w", err) + return nil, errors.Wrap(err, "reset creds") } return metabase, nil @@ -193,7 +193,7 @@ func (m *Metabase) WaitAlive() error { if err != nil { if strings.Contains(err.Error(), "password:did not match stored password") { log.Errorf("Password mismatch error, is your dashboard already setup ? Run 'cscli dashboard remove' to reset it.") - return fmt.Errorf("password mismatch error: %w", err) + return errors.Wrapf(err, "Password mismatch error") } log.Debugf("%+v", err) } else { @@ -215,7 +215,7 @@ func (m *Metabase) Login(username string, password string) error { } if errormsg != nil { - return fmt.Errorf("http login: %s", errormsg) + return errors.Wrap(err, "http login") } resp, ok := successmsg.(map[string]interface{}) if !ok { @@ -238,7 +238,7 @@ func (m *Metabase) Scan() error { return err } if errormsg != nil { - return fmt.Errorf("http scan: %s", errormsg) + return errors.Wrap(err, "http scan") } return nil @@ -252,10 +252,10 @@ func (m *Metabase) ResetPassword(current string, newPassword string) error { } _, errormsg, err := m.Client.Do("PUT", routes[resetPasswordEndpoint], body) if err != nil { - return fmt.Errorf("reset username: %w", err) + return errors.Wrap(err, "reset username") } if errormsg != nil { - return fmt.Errorf("http reset password: %s", errormsg) + return errors.Wrap(err, "http reset password") } return nil } @@ -275,11 +275,11 @@ func (m *Metabase) ResetUsername(username string) error { _, errormsg, err := m.Client.Do("PUT", routes[userEndpoint], body) if err != nil { - return fmt.Errorf("reset username: %w", err) + return errors.Wrap(err, "reset username") } if errormsg != nil { - return fmt.Errorf("http reset username: %s", errormsg) + return errors.Wrap(err, "http reset username") } return nil diff --git a/pkg/parser/enrich.go b/pkg/parser/enrich.go index 5180b9a5f..04652407d 100644 --- a/pkg/parser/enrich.go +++ b/pkg/parser/enrich.go @@ -1,12 +1,11 @@ package parser import ( - log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/crowdsec/pkg/types" + log "github.com/sirupsen/logrus" ) -/* should be part of a package shared with enrich/geoip.go */ +/* should be part of a packaged shared with enrich/geoip.go */ type EnrichFunc func(string, *types.Event, interface{}, *log.Entry) (map[string]string, error) type InitFunc func(map[string]string) (interface{}, error) diff --git a/pkg/parser/enrich_date.go b/pkg/parser/enrich_date.go index 20828af90..a1dd994be 100644 --- a/pkg/parser/enrich_date.go +++ b/pkg/parser/enrich_date.go @@ -3,10 +3,9 @@ package parser import ( "time" - log "github.com/sirupsen/logrus" - expr "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/types" + log "github.com/sirupsen/logrus" ) func parseDateWithFormat(date, format string) (string, time.Time) { diff --git a/pkg/parser/enrich_dns.go b/pkg/parser/enrich_dns.go index f622e6c35..5bbc0e94d 100644 --- a/pkg/parser/enrich_dns.go +++ b/pkg/parser/enrich_dns.go @@ -3,9 +3,9 @@ package parser import ( "net" - log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/crowdsec/pkg/types" + log "github.com/sirupsen/logrus" + //"github.com/crowdsecurity/crowdsec/pkg/parser" ) /* All plugins must export a list of function pointers for exported symbols */ diff --git a/pkg/parser/enrich_geoip.go b/pkg/parser/enrich_geoip.go index 0a263c827..8c25bef44 100644 --- a/pkg/parser/enrich_geoip.go +++ b/pkg/parser/enrich_geoip.go @@ -5,11 +5,11 @@ import ( "net" "strconv" - "github.com/oschwald/geoip2-golang" - "github.com/oschwald/maxminddb-golang" + "github.com/crowdsecurity/crowdsec/pkg/types" log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/oschwald/geoip2-golang" + "github.com/oschwald/maxminddb-golang" ) func IpToRange(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) { diff --git a/pkg/parser/enrich_unmarshal.go b/pkg/parser/enrich_unmarshal.go index dce9c75d4..404340401 100644 --- a/pkg/parser/enrich_unmarshal.go +++ b/pkg/parser/enrich_unmarshal.go @@ -3,9 +3,8 @@ package parser import ( "encoding/json" - log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/crowdsec/pkg/types" + log "github.com/sirupsen/logrus" ) func unmarshalJSON(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) { diff --git a/pkg/parser/node.go b/pkg/parser/node.go index 036a244cc..2370d3796 100644 --- a/pkg/parser/node.go +++ b/pkg/parser/node.go @@ -1,24 +1,24 @@ package parser import ( - "errors" "fmt" "net" "strings" "time" "github.com/antonmedv/expr" - "github.com/antonmedv/expr/vm" - "github.com/davecgh/go-spew/spew" - "github.com/prometheus/client_golang/prometheus" - log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/grokky" + "github.com/pkg/errors" yaml "gopkg.in/yaml.v2" - "github.com/crowdsecurity/grokky" - + "github.com/antonmedv/expr/vm" "github.com/crowdsecurity/crowdsec/pkg/cache" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/davecgh/go-spew/spew" + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + log "github.com/sirupsen/logrus" ) type Node struct { @@ -56,11 +56,11 @@ type Node struct { SubGroks yaml.MapSlice `yaml:"pattern_syntax,omitempty"` //Holds a grok pattern - Grok GrokPattern `yaml:"grok,omitempty"` + Grok types.GrokPattern `yaml:"grok,omitempty"` //Statics can be present in any type of node and is executed last - Statics []ExtraField `yaml:"statics,omitempty"` + Statics []types.ExtraField `yaml:"statics,omitempty"` //Stash allows to capture data from the log line and store it in an accessible cache - Stash []DataCapture `yaml:"stash,omitempty"` + Stash []types.DataCapture `yaml:"stash,omitempty"` //Whitelists Whitelist Whitelist `yaml:"whitelist,omitempty"` Data []*types.DataSource `yaml:"data,omitempty"` @@ -360,27 +360,29 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri } //Iterate on leafs - for _, leaf := range n.LeavesNodes { - ret, err := leaf.process(p, ctx, cachedExprEnv) - if err != nil { - clog.Tracef("\tNode (%s) failed : %v", leaf.rn, err) - clog.Debugf("Event leaving node : ko") - return false, err - } - clog.Tracef("\tsub-node (%s) ret : %v (strategy:%s)", leaf.rn, ret, n.OnSuccess) - if ret { - NodeState = true - /* if child is successful, stop processing */ - if n.OnSuccess == "next_stage" { - clog.Debugf("child is success, OnSuccess=next_stage, skip") - break + if len(n.LeavesNodes) > 0 { + for _, leaf := range n.LeavesNodes { + ret, err := leaf.process(p, ctx, cachedExprEnv) + if err != nil { + clog.Tracef("\tNode (%s) failed : %v", leaf.rn, err) + clog.Debugf("Event leaving node : ko") + return false, err + } + clog.Tracef("\tsub-node (%s) ret : %v (strategy:%s)", leaf.rn, ret, n.OnSuccess) + if ret { + NodeState = true + /* if child is successful, stop processing */ + if n.OnSuccess == "next_stage" { + clog.Debugf("child is success, OnSuccess=next_stage, skip") + break + } + } else if !NodeHasOKGrok { + /* + If the parent node has a successful grok pattern, it's state will stay successful even if one or more chil fails. + If the parent node is a skeleton node (no grok pattern), then at least one child must be successful for it to be a success. + */ + NodeState = false } - } else if !NodeHasOKGrok { - /* - If the parent node has a successful grok pattern, it's state will stay successful even if one or more chil fails. - If the parent node is a skeleton node (no grok pattern), then at least one child must be successful for it to be a success. - */ - NodeState = false } } /*todo : check if a node made the state change ?*/ @@ -400,11 +402,10 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri if n.Name != "" { NodesHitsOk.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc() } - /* - This is to apply statics when the node *has* whitelists that successfully matched the node. + Please kill me. this is to apply statics when the node *has* whitelists that successfully matched the node. */ - if len(n.Statics) > 0 && (isWhitelisted || !hasWhitelist) { + if hasWhitelist && isWhitelisted && len(n.Statics) > 0 || len(n.Statics) > 0 && !hasWhitelist { clog.Debugf("+ Processing %d statics", len(n.Statics)) // if all else is good in whitelist, process node's statics err := n.ProcessStatics(n.Statics, p) @@ -452,8 +453,8 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { /* if the node has debugging enabled, create a specific logger with debug that will be used only for processing this node ;) */ if n.Debug { - var clog = log.New() - if err = types.ConfigureLogger(clog); err != nil { + var clog = logrus.New() + if err := types.ConfigureLogger(clog); err != nil { log.Fatalf("While creating bucket-specific logger : %s", err) } clog.SetLevel(log.DebugLevel) @@ -492,7 +493,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { /* handle pattern_syntax and groks */ for _, pattern := range n.SubGroks { n.Logger.Tracef("Adding subpattern '%s' : '%s'", pattern.Key, pattern.Value) - if err = pctx.Grok.Add(pattern.Key.(string), pattern.Value.(string)); err != nil { + if err := pctx.Grok.Add(pattern.Key.(string), pattern.Value.(string)); err != nil { if errors.Is(err, grokky.ErrAlreadyExist) { n.Logger.Warningf("grok '%s' already registred", pattern.Key) continue @@ -535,18 +536,20 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { n.Grok.RunTimeValue, err = expr.Compile(n.Grok.ExpValue, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) if err != nil { - return fmt.Errorf("while compiling grok's expression: %w", err) + return errors.Wrap(err, "while compiling grok's expression") } } /* load grok statics */ - //compile expr statics if present - for idx := range n.Grok.Statics { - if n.Grok.Statics[idx].ExpValue != "" { - n.Grok.Statics[idx].RunTimeValue, err = expr.Compile(n.Grok.Statics[idx].ExpValue, - exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) - if err != nil { - return err + if len(n.Grok.Statics) > 0 { + //compile expr statics if present + for idx := range n.Grok.Statics { + if n.Grok.Statics[idx].ExpValue != "" { + n.Grok.Statics[idx].RunTimeValue, err = expr.Compile(n.Grok.Statics[idx].ExpValue, + exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) + if err != nil { + return err + } } } valid = true @@ -557,53 +560,54 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { n.Stash[i].ValueExpression, err = expr.Compile(stash.Value, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) if err != nil { - return fmt.Errorf("while compiling stash value expression: %w", err) + return errors.Wrap(err, "while compiling stash value expression") } n.Stash[i].KeyExpression, err = expr.Compile(stash.Key, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) if err != nil { - return fmt.Errorf("while compiling stash key expression: %w", err) + return errors.Wrap(err, "while compiling stash key expression") } n.Stash[i].TTLVal, err = time.ParseDuration(stash.TTL) if err != nil { - return fmt.Errorf("while parsing stash ttl: %w", err) + return errors.Wrap(err, "while parsing stash ttl") } logLvl := n.Logger.Logger.GetLevel() //init the cache, does it make sense to create it here just to be sure everything is fine ? - if err = cache.CacheInit(cache.CacheCfg{ + if err := cache.CacheInit(cache.CacheCfg{ Size: n.Stash[i].MaxMapSize, TTL: n.Stash[i].TTLVal, Name: n.Stash[i].Name, Strategy: n.Stash[i].Strategy, LogLevel: &logLvl, }); err != nil { - return fmt.Errorf("while initializing cache: %w", err) + return errors.Wrap(err, "while initializing cache") } } /* compile leafs if present */ - for idx := range n.LeavesNodes { - if n.LeavesNodes[idx].Name == "" { - n.LeavesNodes[idx].Name = fmt.Sprintf("child-%s", n.Name) - } - /*propagate debug/stats to child nodes*/ - if !n.LeavesNodes[idx].Debug && n.Debug { - n.LeavesNodes[idx].Debug = true - } - if !n.LeavesNodes[idx].Profiling && n.Profiling { - n.LeavesNodes[idx].Profiling = true - } - n.LeavesNodes[idx].Stage = n.Stage - err = n.LeavesNodes[idx].compile(pctx, ectx) - if err != nil { - return err + if len(n.LeavesNodes) > 0 { + for idx := range n.LeavesNodes { + if n.LeavesNodes[idx].Name == "" { + n.LeavesNodes[idx].Name = fmt.Sprintf("child-%s", n.Name) + } + /*propagate debug/stats to child nodes*/ + if !n.LeavesNodes[idx].Debug && n.Debug { + n.LeavesNodes[idx].Debug = true + } + if !n.LeavesNodes[idx].Profiling && n.Profiling { + n.LeavesNodes[idx].Profiling = true + } + n.LeavesNodes[idx].Stage = n.Stage + err = n.LeavesNodes[idx].compile(pctx, ectx) + if err != nil { + return err + } } valid = true } - /* load statics if present */ for idx := range n.Statics { if n.Statics[idx].ExpValue != "" { @@ -622,7 +626,6 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { n.Logger.Debugf("adding ip %s to whitelists", net.ParseIP(v)) valid = true } - for _, v := range n.Whitelist.Cidrs { _, tnet, err := net.ParseCIDR(v) if err != nil { @@ -632,7 +635,6 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { n.Logger.Debugf("adding cidr %s to whitelists", tnet) valid = true } - for _, filter := range n.Whitelist.Exprs { expression := &ExprWhitelist{} expression.Filter, err = expr.Compile(filter, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) @@ -658,6 +660,5 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { if err := n.validate(pctx, ectx); err != nil { return err } - return nil } diff --git a/pkg/parser/node_test.go b/pkg/parser/node_test.go index d85aa82a8..f529cb4d4 100644 --- a/pkg/parser/node_test.go +++ b/pkg/parser/node_test.go @@ -3,6 +3,7 @@ package parser import ( "testing" + "github.com/crowdsecurity/crowdsec/pkg/types" yaml "gopkg.in/yaml.v2" ) @@ -19,7 +20,7 @@ func TestParserConfigs(t *testing.T) { Valid bool }{ //valid node with grok pattern - {&Node{Debug: true, Stage: "s00", Grok: GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true}, + {&Node{Debug: true, Stage: "s00", Grok: types.GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true}, //bad filter {&Node{Debug: true, Stage: "s00", Filter: "ratata"}, false, false}, //empty node @@ -27,25 +28,25 @@ func TestParserConfigs(t *testing.T) { //bad subgrok {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBAR"), Value: string("[a-$")}}}, false, true}, //valid node with grok pattern - {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBAR"), Value: string("[a-z]")}}, Grok: GrokPattern{RegexpValue: "^x%{FOOBAR:extr}$", TargetField: "t"}}, true, true}, + {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBAR"), Value: string("[a-z]")}}, Grok: types.GrokPattern{RegexpValue: "^x%{FOOBAR:extr}$", TargetField: "t"}}, true, true}, //bad node success - {&Node{Debug: true, Stage: "s00", OnSuccess: "ratat", Grok: GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, false, false}, + {&Node{Debug: true, Stage: "s00", OnSuccess: "ratat", Grok: types.GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, false, false}, //ok node success - {&Node{Debug: true, Stage: "s00", OnSuccess: "continue", Grok: GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true}, + {&Node{Debug: true, Stage: "s00", OnSuccess: "continue", Grok: types.GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true}, //valid node with grok sub-pattern used by name - {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBARx"), Value: string("[a-z] %{DATA:lol}$")}}, Grok: GrokPattern{RegexpName: "FOOBARx", TargetField: "t"}}, true, true}, + {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBARx"), Value: string("[a-z] %{DATA:lol}$")}}, Grok: types.GrokPattern{RegexpName: "FOOBARx", TargetField: "t"}}, true, true}, //node with unexisting grok pattern - {&Node{Debug: true, Stage: "s00", Grok: GrokPattern{RegexpName: "RATATA", TargetField: "t"}}, false, true}, + {&Node{Debug: true, Stage: "s00", Grok: types.GrokPattern{RegexpName: "RATATA", TargetField: "t"}}, false, true}, //node with grok pattern dependencies {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{ {Key: string("SUBGROK"), Value: string("[a-z]")}, {Key: string("MYGROK"), Value: string("[a-z]%{SUBGROK}")}, - }, Grok: GrokPattern{RegexpValue: "^x%{MYGROK:extr}$", TargetField: "t"}}, true, true}, + }, Grok: types.GrokPattern{RegexpValue: "^x%{MYGROK:extr}$", TargetField: "t"}}, true, true}, //node with broken grok pattern dependencies {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{ {Key: string("SUBGROKBIS"), Value: string("[a-z]%{MYGROKBIS}")}, {Key: string("MYGROKBIS"), Value: string("[a-z]")}, - }, Grok: GrokPattern{RegexpValue: "^x%{MYGROKBIS:extr}$", TargetField: "t"}}, false, true}, + }, Grok: types.GrokPattern{RegexpValue: "^x%{MYGROKBIS:extr}$", TargetField: "t"}}, false, true}, } for idx := range CfgTests { err := CfgTests[idx].NodeCfg.compile(pctx, EnricherCtx{}) @@ -63,5 +64,6 @@ func TestParserConfigs(t *testing.T) { if CfgTests[idx].Valid == false && err == nil { t.Fatalf("Valid: (%d/%d) expected error", idx+1, len(CfgTests)) } + } } diff --git a/pkg/parser/parsing_test.go b/pkg/parser/parsing_test.go index 04d08cc27..089b95835 100644 --- a/pkg/parser/parsing_test.go +++ b/pkg/parser/parsing_test.go @@ -11,12 +11,11 @@ import ( "strings" "testing" + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/davecgh/go-spew/spew" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" - - "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" - "github.com/crowdsecurity/crowdsec/pkg/types" ) type TestFile struct { @@ -55,6 +54,7 @@ func TestParser(t *testing.T) { } } } + } func BenchmarkParser(t *testing.B) { @@ -90,6 +90,7 @@ func BenchmarkParser(t *testing.B) { } func testOneParser(pctx *UnixParserCtx, ectx EnricherCtx, dir string, b *testing.B) error { + var ( err error pnodes []Node @@ -111,7 +112,7 @@ func testOneParser(pctx *UnixParserCtx, ectx EnricherCtx, dir string, b *testing if err != nil { panic(err) } - if err = yaml.UnmarshalStrict(out.Bytes(), &parser_configs); err != nil { + if err := yaml.UnmarshalStrict(out.Bytes(), &parser_configs); err != nil { return fmt.Errorf("failed unmarshaling %s : %s", parser_cfg_file, err) } @@ -398,7 +399,7 @@ func TestGeneratePatternsDoc(t *testing.T) { t.Fatal("failed to write to file") } for _, k := range p { - if _, err := fmt.Fprintf(f, "## %s\n\nPattern :\n```\n%s\n```\n\n", k.Key, k.Value); err != nil { + if _, err := f.WriteString(fmt.Sprintf("## %s\n\nPattern :\n```\n%s\n```\n\n", k.Key, k.Value)); err != nil { t.Fatal("failed to write to file") } fmt.Printf("%v\t%v\n", k.Key, k.Value) @@ -413,4 +414,5 @@ func TestGeneratePatternsDoc(t *testing.T) { t.Fatal("failed to write to file") } f.Close() + } diff --git a/pkg/parser/runtime.go b/pkg/parser/runtime.go index e6ee07ea7..3c7d382d5 100644 --- a/pkg/parser/runtime.go +++ b/pkg/parser/runtime.go @@ -9,21 +9,24 @@ import ( "errors" "fmt" "reflect" - "strconv" "strings" "sync" "time" - "github.com/antonmedv/expr" + "github.com/crowdsecurity/crowdsec/pkg/types" + + "strconv" + "github.com/mohae/deepcopy" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/antonmedv/expr" ) /* ok, this is kinda experimental, I don't know how bad of an idea it is .. */ func SetTargetByName(target string, value string, evt *types.Event) bool { + if evt == nil { return false } @@ -68,6 +71,8 @@ func SetTargetByName(target string, value string, evt *types.Event) bool { tmp = reflect.Indirect(tmp) } iter = tmp + //nolint: gosimple + break case reflect.Ptr: tmp := iter.Elem() iter = reflect.Indirect(tmp.FieldByName(f)) @@ -89,24 +94,24 @@ func SetTargetByName(target string, value string, evt *types.Event) bool { return true } -func printStaticTarget(static ExtraField) string { - switch { - case static.Method != "": +func printStaticTarget(static types.ExtraField) string { + + if static.Method != "" { return static.Method - case static.Parsed != "": + } else if static.Parsed != "" { return fmt.Sprintf(".Parsed[%s]", static.Parsed) - case static.Meta != "": + } else if static.Meta != "" { return fmt.Sprintf(".Meta[%s]", static.Meta) - case static.Enriched != "": + } else if static.Enriched != "" { return fmt.Sprintf(".Enriched[%s]", static.Enriched) - case static.TargetByName != "": + } else if static.TargetByName != "" { return static.TargetByName - default: + } else { return "?" } } -func (n *Node) ProcessStatics(statics []ExtraField, event *types.Event) error { +func (n *Node) ProcessStatics(statics []types.ExtraField, event *types.Event) error { //we have a few cases : //(meta||key) + (static||reference||expr) var value string @@ -192,6 +197,7 @@ func (n *Node) ProcessStatics(statics []ExtraField, event *types.Event) error { } else { clog.Fatal("unable to process static : unknown target") } + } return nil } @@ -351,8 +357,10 @@ func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error) event.Process = false return event, nil } + } event.Process = true return event, nil + } diff --git a/pkg/parser/stage.go b/pkg/parser/stage.go index 37d43fbfe..3bdaed1f0 100644 --- a/pkg/parser/stage.go +++ b/pkg/parser/stage.go @@ -109,16 +109,17 @@ func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx, ectx EnricherCtx) ( continue } - for _, data := range node.Data { - err = exprhelpers.FileInit(pctx.DataFolder, data.DestPath, data.Type) - if err != nil { - log.Error(err) - } - if data.Type == "regexp" { //cache only makes sense for regexp - exprhelpers.RegexpCacheInit(data.DestPath, *data) + if len(node.Data) > 0 { + for _, data := range node.Data { + err = exprhelpers.FileInit(pctx.DataFolder, data.DestPath, data.Type) + if err != nil { + log.Error(err) + } + if data.Type == "regexp" { //cache only makes sense for regexp + exprhelpers.RegexpCacheInit(data.DestPath, *data) + } } } - nodes = append(nodes, node) nodesCount++ } diff --git a/pkg/parser/unix_parser.go b/pkg/parser/unix_parser.go index d5d91f932..b0f38f81e 100644 --- a/pkg/parser/unix_parser.go +++ b/pkg/parser/unix_parser.go @@ -7,13 +7,12 @@ import ( "sort" "strings" - log "github.com/sirupsen/logrus" - - "github.com/crowdsecurity/grokky" - "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/fflag" + + "github.com/crowdsecurity/grokky" + log "github.com/sirupsen/logrus" ) type UnixParserCtx struct { @@ -118,7 +117,7 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) { parsers.EnricherCtx, err = Loadplugin(cConfig.Crowdsec.DataDir) if err != nil { - return parsers, fmt.Errorf("failed to load enrich plugin : %v", err) + return parsers, fmt.Errorf("Failed to load enrich plugin : %v", err) } /* @@ -136,8 +135,8 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) { log.Infof("Loading postoverflow parsers") parsers.Povfwnodes, err = LoadStages(parsers.PovfwStageFiles, parsers.Povfwctx, parsers.EnricherCtx) } else { - log.Infof("No postoverflow parsers to load") parsers.Povfwnodes = []Node{} + log.Infof("No postoverflow parsers to load") } if err != nil { diff --git a/pkg/parser/whitelist.go b/pkg/parser/whitelist.go index e2f179fb3..c56ad40a7 100644 --- a/pkg/parser/whitelist.go +++ b/pkg/parser/whitelist.go @@ -4,7 +4,6 @@ import ( "net" "github.com/antonmedv/expr/vm" - "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" ) diff --git a/pkg/setup/detect.go b/pkg/setup/detect.go index b345c0d6f..957f70a25 100644 --- a/pkg/setup/detect.go +++ b/pkg/setup/detect.go @@ -7,12 +7,15 @@ import ( "os/exec" "sort" - "github.com/Masterminds/semver/v3" + "github.com/Masterminds/semver" "github.com/antonmedv/expr" "github.com/blackfireio/osinfo" "github.com/shirou/gopsutil/v3/process" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" + // goccyyaml "github.com/goccy/go-yaml" + + // "github.com/k0kubun/pp" "github.com/crowdsecurity/crowdsec/pkg/acquisition" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" @@ -49,6 +52,7 @@ func validateDataSource(opaqueDS DataSourceItem) error { return nil } + // formally validate YAML commonDS := configuration.DataSourceCommonCfg{} @@ -68,6 +72,7 @@ func validateDataSource(opaqueDS DataSourceItem) error { return fmt.Errorf("source is empty") } + // source must be known ds := acquisition.GetDataSourceIface(commonDS.Source) diff --git a/pkg/setup/detect_test.go b/pkg/setup/detect_test.go index adb5f1d43..4f6ef0c33 100644 --- a/pkg/setup/detect_test.go +++ b/pkg/setup/detect_test.go @@ -10,7 +10,6 @@ import ( "github.com/lithammer/dedent" "github.com/stretchr/testify/require" - "github.com/crowdsecurity/go-cs-lib/pkg/csstring" "github.com/crowdsecurity/go-cs-lib/pkg/cstest" "github.com/crowdsecurity/crowdsec/pkg/setup" @@ -118,16 +117,16 @@ func TestVersionCheck(t *testing.T) { {"1", ">1", false, ""}, {"1", ">=1", true, ""}, {"1.0", "<1.0", false, ""}, - {"1", "<1", false, ""}, - {"1.3.5", "1.3", true, ""}, + {"1", "<1", true, ""}, // XXX why? + {"1.3.5", "1.3", false, ""}, // XXX ok? {"1.0", "<1.0", false, ""}, {"1.0", "<=1.0", true, ""}, {"2", ">1, <3", true, ""}, {"2", "<=2, >=2.2", false, ""}, {"2.3", "~2", true, ""}, {"2.3", "=2", true, ""}, - {"1.1.1", "=1.1", true, ""}, - {"1.1.1", "1.1", true, ""}, + {"1.1.1", "=1.1", false, ""}, + {"1.1.1", "1.1", false, ""}, {"1.1", "!=1.1.1", true, ""}, {"1.1", "~1.1.1", false, ""}, {"1.1.1", "~1.1", true, ""}, @@ -137,7 +136,7 @@ func TestVersionCheck(t *testing.T) { {"19.04", "=19.4", true, ""}, {"19.04", "~19.4", true, ""}, {"1.2.3", "~1.2", true, ""}, - {"1.2.3", "!=1.2", false, ""}, + {"1.2.3", "!=1.2", true, ""}, {"1.2.3", "1.1.1 - 1.3.4", true, ""}, {"1.3.5", "1.1.1 - 1.3.4", false, ""}, {"1.3.5", "=1", true, ""}, @@ -1008,7 +1007,7 @@ func TestDetectDatasourceValidation(t *testing.T) { "DetectYaml": detectYaml, } - expectedErr, err := csstring.Interpolate(tc.expectedErr, data) + expectedErr, err := cstest.Interpolate(tc.expectedErr, data) require.NoError(err) detected, err := setup.Detect(detectYaml, setup.DetectOptions{}) diff --git a/pkg/cwhub/dataset.go b/pkg/types/dataset.go similarity index 70% rename from pkg/cwhub/dataset.go rename to pkg/types/dataset.go index 848686be6..2684342a9 100644 --- a/pkg/cwhub/dataset.go +++ b/pkg/types/dataset.go @@ -1,4 +1,4 @@ -package cwhub +package types import ( "fmt" @@ -6,14 +6,24 @@ import ( "net/http" "os" "path" + "time" log "github.com/sirupsen/logrus" - - "github.com/crowdsecurity/crowdsec/pkg/types" ) +type DataSource struct { + SourceURL string `yaml:"source_url"` + DestPath string `yaml:"dest_file"` + Type string `yaml:"type"` + //Control cache strategy on expensive regexps + Cache *bool `yaml:"cache"` + Strategy *string `yaml:"strategy"` + Size *int `yaml:"size"` + TTL *time.Duration `yaml:"ttl"` +} + type DataSet struct { - Data []*types.DataSource `yaml:"data,omitempty"` + Data []*DataSource `yaml:"data,omitempty"` } func downloadFile(url string, destPath string) error { @@ -56,7 +66,7 @@ func downloadFile(url string, destPath string) error { return nil } -func GetData(data []*types.DataSource, dataDir string) error { +func GetData(data []*DataSource, dataDir string) error { for _, dataS := range data { destPath := path.Join(dataDir, dataS.DestPath) log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath) diff --git a/pkg/cwhub/dataset_test.go b/pkg/types/dataset_test.go similarity index 94% rename from pkg/cwhub/dataset_test.go rename to pkg/types/dataset_test.go index 106268c01..956e3316f 100644 --- a/pkg/cwhub/dataset_test.go +++ b/pkg/types/dataset_test.go @@ -1,4 +1,4 @@ -package cwhub +package types import ( "os" @@ -9,7 +9,7 @@ import ( "github.com/jarcoal/httpmock" ) -func TestDownloadFile(t *testing.T) { +func TestDownladFile(t *testing.T) { examplePath := "./example.txt" defer os.Remove(examplePath) diff --git a/pkg/types/datasource.go b/pkg/types/datasource.go deleted file mode 100644 index 39b83fbaf..000000000 --- a/pkg/types/datasource.go +++ /dev/null @@ -1,16 +0,0 @@ -package types - -import ( - "time" -) - -type DataSource struct { - SourceURL string `yaml:"source_url"` - DestPath string `yaml:"dest_file"` - Type string `yaml:"type"` - //Control cache strategy on expensive regexps - Cache *bool `yaml:"cache"` - Strategy *string `yaml:"strategy"` - Size *int `yaml:"size"` - TTL *time.Duration `yaml:"ttl"` -} diff --git a/pkg/parser/grok_pattern.go b/pkg/types/grok_pattern.go similarity index 99% rename from pkg/parser/grok_pattern.go rename to pkg/types/grok_pattern.go index 5b3204a42..bb51d2d9b 100644 --- a/pkg/parser/grok_pattern.go +++ b/pkg/types/grok_pattern.go @@ -1,10 +1,9 @@ -package parser +package types import ( "time" "github.com/antonmedv/expr/vm" - "github.com/crowdsecurity/grokky" ) diff --git a/pkg/types/ip.go b/pkg/types/ip.go index 5e4d7734f..647fb4a63 100644 --- a/pkg/types/ip.go +++ b/pkg/types/ip.go @@ -6,14 +6,13 @@ import ( "math" "net" "strings" + + "github.com/pkg/errors" ) -// LastAddress returns the last address of a network func LastAddress(n net.IPNet) net.IP { - // get the last address by ORing the hostmask and the IP ip := n.IP.To4() if ip == nil { - // IPv6 ip = n.IP return net.IP{ ip[0] | ^n.Mask[0], ip[1] | ^n.Mask[1], ip[2] | ^n.Mask[2], @@ -36,7 +35,7 @@ func Addr2Ints(anyIP string) (int, int64, int64, int64, int64, error) { if strings.Contains(anyIP, "/") { _, net, err := net.ParseCIDR(anyIP) if err != nil { - return -1, 0, 0, 0, 0, fmt.Errorf("while parsing range %s: %w", anyIP, err) + return -1, 0, 0, 0, 0, errors.Wrapf(err, "while parsing range %s", anyIP) } return Range2Ints(*net) } @@ -48,7 +47,7 @@ func Addr2Ints(anyIP string) (int, int64, int64, int64, int64, error) { sz, start, end, err := IP2Ints(ip) if err != nil { - return -1, 0, 0, 0, 0, fmt.Errorf("while parsing ip %s: %w", anyIP, err) + return -1, 0, 0, 0, 0, errors.Wrapf(err, "while parsing ip %s", anyIP) } return sz, start, end, start, end, nil @@ -59,12 +58,12 @@ func Range2Ints(network net.IPNet) (int, int64, int64, int64, int64, error) { szStart, nwStart, sfxStart, err := IP2Ints(network.IP) if err != nil { - return -1, 0, 0, 0, 0, fmt.Errorf("converting first ip in range: %w", err) + return -1, 0, 0, 0, 0, errors.Wrap(err, "converting first ip in range") } lastAddr := LastAddress(network) szEnd, nwEnd, sfxEnd, err := IP2Ints(lastAddr) if err != nil { - return -1, 0, 0, 0, 0, fmt.Errorf("transforming last address of range: %w", err) + return -1, 0, 0, 0, 0, errors.Wrap(err, "transforming last address of range") } if szEnd != szStart { return -1, 0, 0, 0, 0, fmt.Errorf("inconsistent size for range first(%d) and last(%d) ip", szStart, szEnd) diff --git a/pkg/types/utils.go b/pkg/types/utils.go index 0485db59e..caed6961e 100644 --- a/pkg/types/utils.go +++ b/pkg/types/utils.go @@ -2,9 +2,13 @@ package types import ( "bufio" + "bytes" + "encoding/gob" "fmt" + "io" "os" "path/filepath" + "regexp" "strconv" "strings" "time" @@ -36,9 +40,19 @@ func SetDefaultLoggerConfig(cfgMode string, cfgFolder string, cfgLevel log.Level if compress != nil { _compress = *compress } + /*cf. https://github.com/natefinch/lumberjack/issues/82 + let's create the file beforehand w/ the right perms */ + fname := cfgFolder + "/crowdsec.log" + // check if file exists + _, err := os.Stat(fname) + // create file if not exists, purposefully ignore errors + if os.IsNotExist(err) { + file, _ := os.OpenFile(fname, os.O_RDWR|os.O_CREATE, 0600) + file.Close() + } LogOutput = &lumberjack.Logger{ - Filename: filepath.Join(cfgFolder, "crowdsec.log"), + Filename: fname, MaxSize: _maxsize, MaxBackups: _maxfiles, MaxAge: _maxage, @@ -68,6 +82,19 @@ func ConfigureLogger(clog *log.Logger) error { return nil } +func Clone(a, b interface{}) error { + buff := new(bytes.Buffer) + enc := gob.NewEncoder(buff) + dec := gob.NewDecoder(buff) + if err := enc.Encode(a); err != nil { + return fmt.Errorf("failed cloning %T", a) + } + if err := dec.Decode(b); err != nil { + return fmt.Errorf("failed cloning %T", b) + } + return nil +} + func ParseDuration(d string) (time.Duration, error) { durationStr := d if strings.HasSuffix(d, "d") { @@ -88,6 +115,67 @@ func ParseDuration(d string) (time.Duration, error) { return duration, nil } +/*help to copy the file, ioutil doesn't offer the feature*/ + +func copyFileContents(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return + } + defer func() { + cerr := out.Close() + if err == nil { + err = cerr + } + }() + if _, err = io.Copy(out, in); err != nil { + return + } + err = out.Sync() + return +} + +/*copy the file, ioutile doesn't offer the feature*/ +func CopyFile(sourceSymLink, destinationFile string) (err error) { + sourceFile, err := filepath.EvalSymlinks(sourceSymLink) + if err != nil { + log.Infof("Not a symlink : %s", err) + sourceFile = sourceSymLink + } + + sourceFileStat, err := os.Stat(sourceFile) + if err != nil { + return + } + if !sourceFileStat.Mode().IsRegular() { + // cannot copy non-regular files (e.g., directories, + // symlinks, devices, etc.) + return fmt.Errorf("copyFile: non-regular source file %s (%q)", sourceFileStat.Name(), sourceFileStat.Mode().String()) + } + destinationFileStat, err := os.Stat(destinationFile) + if err != nil { + if !os.IsNotExist(err) { + return + } + } else { + if !(destinationFileStat.Mode().IsRegular()) { + return fmt.Errorf("copyFile: non-regular destination file %s (%q)", destinationFileStat.Name(), destinationFileStat.Mode().String()) + } + if os.SameFile(sourceFileStat, destinationFileStat) { + return + } + } + if err = os.Link(sourceFile, destinationFile); err != nil { + err = copyFileContents(sourceFile, destinationFile) + } + return +} + func UtcNow() time.Time { return time.Now().UTC() } @@ -105,3 +193,11 @@ func GetLineCountForFile(filepath string) int { } return lc } + +// from https://github.com/acarl005/stripansi +var reStripAnsi = regexp.MustCompile("[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))") + +func StripAnsiString(str string) string { + // the byte version doesn't strip correctly + return reStripAnsi.ReplaceAllString(str, "") +} diff --git a/plugins/notifications/dummy/Makefile b/plugins/notifications/dummy/Makefile index d45d6f198..612ec6c86 100644 --- a/plugins/notifications/dummy/Makefile +++ b/plugins/notifications/dummy/Makefile @@ -16,3 +16,8 @@ build: clean .PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) + +.PHONY: vendor +vendor: + @echo "vendoring $(PLUGIN) plugin..." + @$(GOCMD) mod vendor diff --git a/plugins/notifications/email/Makefile b/plugins/notifications/email/Makefile index ae548af0a..a386625ac 100644 --- a/plugins/notifications/email/Makefile +++ b/plugins/notifications/email/Makefile @@ -16,3 +16,8 @@ build: clean .PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) + +.PHONY: vendor +vendor: + @echo "vendoring $(PLUGIN) plugin..." + @$(GOCMD) mod vendor diff --git a/plugins/notifications/email/go.mod b/plugins/notifications/email/go.mod index bfeb5b023..549249467 100644 --- a/plugins/notifications/email/go.mod +++ b/plugins/notifications/email/go.mod @@ -13,16 +13,16 @@ require ( ) require ( - github.com/fatih/color v1.15.0 // indirect + github.com/fatih/color v1.13.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-colorable v0.1.12 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/email/go.sum b/plugins/notifications/email/go.sum index 8077850de..ace03e73b 100644 --- a/plugins/notifications/email/go.sum +++ b/plugins/notifications/email/go.sum @@ -21,8 +21,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -62,13 +62,14 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -83,7 +84,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/xhit/go-simple-mail/v2 v2.10.0 h1:nib6RaJ4qVh5HD9UE9QJqnUZyWp3upv+Z6CFxaMj0V8= github.com/xhit/go-simple-mail/v2 v2.10.0/go.mod h1:kA1XbQfCI4JxQ9ccSN6VFyIEkkugOm7YiPkA5hKiQn4= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= @@ -116,15 +117,18 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/plugins/notifications/http/Makefile b/plugins/notifications/http/Makefile index 56f490772..44ee8c58f 100644 --- a/plugins/notifications/http/Makefile +++ b/plugins/notifications/http/Makefile @@ -16,3 +16,8 @@ build: clean .PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) + +.PHONY: vendor +vendor: + @echo "vendoring $(PLUGIN) plugin..." + @$(GOCMD) mod vendor diff --git a/plugins/notifications/http/go.mod b/plugins/notifications/http/go.mod index 5f599674e..a09f74276 100644 --- a/plugins/notifications/http/go.mod +++ b/plugins/notifications/http/go.mod @@ -12,16 +12,16 @@ require ( ) require ( - github.com/fatih/color v1.15.0 // indirect + github.com/fatih/color v1.13.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-colorable v0.1.12 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/http/go.sum b/plugins/notifications/http/go.sum index c7ba83755..e160060b1 100644 --- a/plugins/notifications/http/go.sum +++ b/plugins/notifications/http/go.sum @@ -21,8 +21,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -62,13 +62,14 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -83,7 +84,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -114,15 +115,18 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/plugins/notifications/slack/Makefile b/plugins/notifications/slack/Makefile index f43303eb8..e950eba92 100644 --- a/plugins/notifications/slack/Makefile +++ b/plugins/notifications/slack/Makefile @@ -16,3 +16,8 @@ build: clean .PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) + +.PHONY: vendor +vendor: + @echo "vendoring $(PLUGIN) plugin..." + @$(GOCMD) mod vendor diff --git a/plugins/notifications/slack/go.mod b/plugins/notifications/slack/go.mod index f8ef3c801..92640aad1 100644 --- a/plugins/notifications/slack/go.mod +++ b/plugins/notifications/slack/go.mod @@ -13,18 +13,18 @@ require ( ) require ( - github.com/fatih/color v1.15.0 // indirect + github.com/fatih/color v1.13.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-colorable v0.1.12 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect github.com/pkg/errors v0.9.1 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/slack/go.sum b/plugins/notifications/slack/go.sum index d28c26c3f..d5f30d5e7 100644 --- a/plugins/notifications/slack/go.sum +++ b/plugins/notifications/slack/go.sum @@ -21,8 +21,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho= github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= @@ -66,13 +66,14 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -92,7 +93,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -123,15 +124,18 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/plugins/notifications/splunk/Makefile b/plugins/notifications/splunk/Makefile index a7f04f4d0..a49c87bd6 100644 --- a/plugins/notifications/splunk/Makefile +++ b/plugins/notifications/splunk/Makefile @@ -16,3 +16,8 @@ build: clean .PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) + +.PHONY: vendor +vendor: + @echo "vendoring $(PLUGIN) plugin..." + @$(GOCMD) mod vendor diff --git a/plugins/notifications/splunk/go.mod b/plugins/notifications/splunk/go.mod index e2d45c365..04e4c3ccc 100644 --- a/plugins/notifications/splunk/go.mod +++ b/plugins/notifications/splunk/go.mod @@ -12,16 +12,16 @@ require ( ) require ( - github.com/fatih/color v1.15.0 // indirect + github.com/fatih/color v1.13.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-colorable v0.1.12 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/splunk/go.sum b/plugins/notifications/splunk/go.sum index c7ba83755..e160060b1 100644 --- a/plugins/notifications/splunk/go.sum +++ b/plugins/notifications/splunk/go.sum @@ -21,8 +21,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -62,13 +62,14 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -83,7 +84,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -114,15 +115,18 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/rpm/SPECS/crowdsec.spec b/rpm/SPECS/crowdsec.spec index a57492eea..246c7b181 100644 --- a/rpm/SPECS/crowdsec.spec +++ b/rpm/SPECS/crowdsec.spec @@ -12,6 +12,8 @@ Patch0: crowdsec.unit.patch Patch1: user.patch BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) +BuildRequires: git +BuildRequires: make BuildRequires: systemd Requires: crontabs %{?fc33:BuildRequires: systemd-rpm-macros} @@ -25,6 +27,7 @@ Requires: crontabs %define version_number %(echo $VERSION) %define releasever %(echo $RELEASEVER) +%global local_version v%{version_number}-%{releasever}-rpm %global name crowdsec %global __mangle_shebangs_exclude_from /usr/bin/env @@ -35,6 +38,7 @@ Requires: crontabs %patch1 %build +BUILD_VERSION=%{local_version} make build sed -i "s#/usr/local/lib/crowdsec/plugins/#%{_libdir}/%{name}/plugins/#g" config/config.yaml %install diff --git a/test/README.md b/test/README.md index 7f34bd3db..bbc40a388 100644 --- a/test/README.md +++ b/test/README.md @@ -56,6 +56,9 @@ architectures. ## pre-requisites - `git submodule init; git submodule update` + - `go install github.com/cloudflare/cfssl/cmd/cfssl@latest` + - `go install github.com/cloudflare/cfssl/cmd/cfssljson@latest` + - `go install github.com/mikefarah/yq/v4@latest` - `base64` - `bash>=4.4` - `curl` @@ -86,7 +89,7 @@ In BATS, you write tests in the form of Bash functions that have unique descriptions (the name of the test). You can do most things that you can normally do in a shell function. If there is any error condition, the test fails. A set of functions is provided to implement assertions, and a mechanism -of `setup`/`teardown` is provided at the level of individual tests (functions) +of `setup`/`teardown` is provided a the level of individual tests (functions) or group of tests (files). The stdout/stderr of the commands within the test function are captured by @@ -126,6 +129,11 @@ included in a larger test suite. The TAP specification is pretty minimalist and some glue may be needed. +Other tools that you can find useful: + + - [mikefarah/yq](https://github.com/mikefarah/yq) - to parse and update YAML files on the fly + - [aliou/bats.vim](https://github.com/aliou/bats.vim) - for syntax highlighting (use bash otherwise) + # setup and teardown If you have read the bats-core tutorial linked above, you are aware of the diff --git a/test/ansible/debug_tools.yml b/test/ansible/debug_tools.yml index 769a973fe..15baa7cab 100644 --- a/test/ansible/debug_tools.yml +++ b/test/ansible/debug_tools.yml @@ -14,5 +14,3 @@ - zsh-autosuggestions - zsh-syntax-highlighting - zsh-theme-powerlevel9k - when: - - ansible_facts.os_family == "Debian" diff --git a/test/ansible/provision_dependencies.yml b/test/ansible/provision_dependencies.yml index bcfe8fcca..891bcc16e 100644 --- a/test/ansible/provision_dependencies.yml +++ b/test/ansible/provision_dependencies.yml @@ -13,8 +13,6 @@ - crowdsecurity.testing.git - crowdsecurity.testing.gcc - crowdsecurity.testing.gnu_make - - crowdsecurity.testing.pkg_config - - crowdsecurity.testing.re2 - crowdsecurity.testing.bats_requirements - name: "Install Postgres" diff --git a/test/ansible/requirements.yml b/test/ansible/requirements.yml index 70f8ca394..b1a28b70a 100644 --- a/test/ansible/requirements.yml +++ b/test/ansible/requirements.yml @@ -11,11 +11,11 @@ roles: - src: gantsign.golang collections: - - name: ansible.posix - name: https://github.com/crowdsecurity/ansible-collection-crowdsecurity.testing.git type: git - version: v0.0.4 + version: main # - name: crowdsecurity.testing # source: ../../../crowdsecurity.testing # type: dir + diff --git a/test/ansible/roles/make_fixture/tasks/main.yml b/test/ansible/roles/make_fixture/tasks/main.yml index 305cec3a6..39f3e1785 100644 --- a/test/ansible/roles/make_fixture/tasks/main.yml +++ b/test/ansible/roles/make_fixture/tasks/main.yml @@ -1,6 +1,5 @@ # vim: set ft=yaml.ansible: --- - - name: "Set make_cmd = make (!bsd)" ansible.builtin.set_fact: make_cmd: make @@ -18,8 +17,7 @@ block: - name: "Make bats-build bats-fixture" ansible.builtin.command: - # static build and we don't have to mess with LD_LIBRARY_PATH - cmd: "{{ make_cmd }} bats-build bats-fixture BUILD_STATIC=1" + cmd: "{{ make_cmd }} bats-build bats-fixture" chdir: "{{ ansible_env.HOME }}/crowdsec" creates: "{{ ansible_env.HOME }}/crowdsec/test/local-init/init-config-data.tar" environment: diff --git a/test/ansible/vagrant/alma-8/Vagrantfile b/test/ansible/vagrant/alma-8/Vagrantfile index e246c77ff..4b42adb3a 100644 --- a/test/ansible/vagrant/alma-8/Vagrantfile +++ b/test/ansible/vagrant/alma-8/Vagrantfile @@ -2,10 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/alma8' - config.vm.provision "shell", inline: <<-SHELL - sudo dnf -y install dnf-plugins-core kitty-terminfo - sudo dnf config-manager --set-enabled powertools - SHELL end common = '../common' diff --git a/test/ansible/vagrant/alma-9/Vagrantfile b/test/ansible/vagrant/alma-9/Vagrantfile index 9c3d1b67c..0ac3e5fe3 100644 --- a/test/ansible/vagrant/alma-9/Vagrantfile +++ b/test/ansible/vagrant/alma-9/Vagrantfile @@ -2,10 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/alma9' - config.vm.provision "shell", inline: <<-SHELL - sudo dnf -y install kitty-terminfo - sudo dnf config-manager --set-enabled crb - SHELL end common = '../common' diff --git a/test/ansible/vagrant/centos-7/Vagrantfile b/test/ansible/vagrant/centos-7/Vagrantfile index fd207359f..d7ac021d2 100644 --- a/test/ansible/vagrant/centos-7/Vagrantfile +++ b/test/ansible/vagrant/centos-7/Vagrantfile @@ -2,8 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'centos/7' - config.vm.provision "shell", inline: <<-SHELL - SHELL end common = '../common' diff --git a/test/ansible/vagrant/centos-8/Vagrantfile b/test/ansible/vagrant/centos-8/Vagrantfile index a88d621c4..24c37ada9 100644 --- a/test/ansible/vagrant/centos-8/Vagrantfile +++ b/test/ansible/vagrant/centos-8/Vagrantfile @@ -1,11 +1,7 @@ # frozen_string_literal: true Vagrant.configure('2') do |config| - config.vm.box = 'generic/centos8s' - config.vm.provision "shell", inline: <<-SHELL - sudo dnf -y install dnf-plugins-core kitty-terminfo - sudo dnf config-manager --set-enabled powertools - SHELL + config.vm.box = 'centos/stream8' end common = '../common' diff --git a/test/ansible/vagrant/centos-9/Vagrantfile b/test/ansible/vagrant/centos-9/Vagrantfile index 1087225be..412354f3d 100644 --- a/test/ansible/vagrant/centos-9/Vagrantfile +++ b/test/ansible/vagrant/centos-9/Vagrantfile @@ -2,10 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/centos9s' - config.vm.provision "shell", inline: <<-SHELL - sudo dnf -y install dnf-plugins-core - sudo dnf config-manager --set-enabled crb - SHELL end common = '../common' diff --git a/test/ansible/vagrant/common b/test/ansible/vagrant/common index 4bc237a7e..adafa08c5 100644 --- a/test/ansible/vagrant/common +++ b/test/ansible/vagrant/common @@ -1,75 +1,46 @@ # vim: set ft=ruby: # frozen_string_literal: true -def find_ansible_cfg - path = Pathname.new(Dir.pwd) - until path.root? - ansible_cfg = path + 'ansible.cfg' - return path if ansible_cfg.exist? - path = path.parent - end - nil # return nil if not found -end - Vagrant.configure('2') do |config| config.vm.define 'crowdsec' - if ARGV.any? { |arg| arg == 'up' || arg == 'provision' } - unless ENV['DB_BACKEND'] - $stderr.puts "\e[31mThe DB_BACKEND environment variable is not defined. Please set up the environment and try again.\e[0m" - exit 1 - end - end - - config.vm.provision 'shell', path: 'bootstrap' if File.exists?('bootstrap') - config.vm.synced_folder '.', '/vagrant', disabled: true - config.vm.provider :libvirt do |libvirt| libvirt.cpus = 1 libvirt.memory = 1536 end - path = find_ansible_cfg - if !path - puts "ansible.cfg not found" - end + config.vm.synced_folder '.', '/vagrant', disabled: true config.vm.provision 'ansible' do |ansible| # ansible.verbose = 'vvvv' - ansible.config_file = (path + 'ansible.cfg').to_s - ansible.playbook = (path + 'run_all.yml').to_s - ansible.compatibility_mode = "2.0" + ansible.config_file = '../../ansible.cfg' + ansible.playbook = '../../run_all.yml' end # same as above, to run the steps separately - # config.vm.provision 'ansible' do |ansible| - # ansible.config_file = (path + 'ansible.cfg').to_s - # ansible.playbook = (path + 'provision_dependencies.yml').to_s - # ansible.compatibility_mode = "2.0" + # config.vm.provision 'ansible' do |provdep| + # provdep.config_file = '../../ansible.cfg' + # provdep.playbook = '../../provision_dependencies.yml' # end - # config.vm.provision 'ansible' do |ansible| - # ansible.config_file = (path + 'ansible.cfg').to_s - # ansible.playbook = (path + 'provision_test_suite.yml').to_s - # ansible.compatibility_mode = "2.0" + # config.vm.provision 'ansible' do |provtest| + # provtest.config_file = '../../ansible.cfg' + # provtest.playbook = '../../provision_test_suite.yml' # end - # config.vm.provision 'ansible' do |ansible| - # ansible.config_file = (path + 'ansible.cfg').to_s - # ansible.playbook = (path + 'install_binary_package.yml').to_s - # ansible.compatibility_mode = "2.0" + # config.vm.provision 'ansible' do |preptest| + # preptest.config_file = '../../ansible.cfg' + # preptest.playbook = '../../install_binary_package.yml' # end - # config.vm.provision 'ansible' do |ansible| - # ansible.config_file = (path + 'ansible.cfg').to_s - # ansible.playbook = (path + 'prepare_tests.yml').to_s - # ansible.compatibility_mode = "2.0" + # config.vm.provision 'ansible' do |preptest| + # preptest.config_file = '../../ansible.cfg' + # preptest.playbook = '../../prepare_tests.yml' # end - # config.vm.provision 'ansible' do |ansible| - # ansible.config_file = (path + 'ansible.cfg').to_s - # ansible.playbook = (path + 'run_tests.yml').to_s - # ansible.compatibility_mode = "2.0" + # config.vm.provision 'ansible' do |runtests| + # runtests.config_file = '../../ansible.cfg' + # runtests.playbook = '../../run_tests.yml' # end end diff --git a/test/ansible/vagrant/debian-10-buster/Vagrantfile b/test/ansible/vagrant/debian-10-buster/Vagrantfile index 8d0a04eaf..2b1a4e2da 100644 --- a/test/ansible/vagrant/debian-10-buster/Vagrantfile +++ b/test/ansible/vagrant/debian-10-buster/Vagrantfile @@ -2,8 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'debian/buster64' - config.vm.provision "shell", inline: <<-SHELL - SHELL end common = '../common' diff --git a/test/ansible/vagrant/debian-11-bullseye/Vagrantfile b/test/ansible/vagrant/debian-11-bullseye/Vagrantfile index fbd2ca41e..9166427cb 100644 --- a/test/ansible/vagrant/debian-11-bullseye/Vagrantfile +++ b/test/ansible/vagrant/debian-11-bullseye/Vagrantfile @@ -2,8 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'debian/bullseye64' - config.vm.provision "shell", inline: <<-SHELL - SHELL end common = '../common' diff --git a/test/ansible/vagrant/debian-12-bookworm/Vagrantfile b/test/ansible/vagrant/debian-12-bookworm/Vagrantfile deleted file mode 100644 index 68a33a947..000000000 --- a/test/ansible/vagrant/debian-12-bookworm/Vagrantfile +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -Vagrant.configure('2') do |config| - config.vm.box = 'debian/bookworm64' - config.vm.provision "shell", inline: <<-SHELL - # sudo apt install -y kitty-terminfo - SHELL -end - -common = '../common' -load common if File.exist?(common) diff --git a/test/ansible/vagrant/debian-9-stretch/Vagrantfile b/test/ansible/vagrant/debian-9-stretch/Vagrantfile index d40b1640c..4c4e39cf7 100644 --- a/test/ansible/vagrant/debian-9-stretch/Vagrantfile +++ b/test/ansible/vagrant/debian-9-stretch/Vagrantfile @@ -2,9 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'debian/stretch64' - config.vm.provision "shell", inline: <<-SHELL - sudo sed -i s/httpredir.debian.org/archive.debian.org/g /etc/apt/sources.list - SHELL end common = '../common' diff --git a/test/ansible/vagrant/debian-testing/Vagrantfile b/test/ansible/vagrant/debian-testing/Vagrantfile index 6cd2667a9..5e3b68e54 100644 --- a/test/ansible/vagrant/debian-testing/Vagrantfile +++ b/test/ansible/vagrant/debian-testing/Vagrantfile @@ -2,8 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'debian/testing64' - config.vm.provision "shell", inline: <<-SHELL - SHELL end common = '../common' diff --git a/test/ansible/vagrant/experimental/hardenedbsd-13/Vagrantfile b/test/ansible/vagrant/experimental/hardenedbsd-13/Vagrantfile index 975d39ea0..0d34ea126 100644 --- a/test/ansible/vagrant/experimental/hardenedbsd-13/Vagrantfile +++ b/test/ansible/vagrant/experimental/hardenedbsd-13/Vagrantfile @@ -2,10 +2,19 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/hardenedbsd13' - config.vm.provision "shell", inline: <<-SHELL - sudo pkg install python3 - SHELL -end + config.vm.define 'crowdsec' -common = '../../common' -load common if File.exist?(common) + config.vm.provision 'shell', path: 'bootstrap' + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 1 + libvirt.memory = 1536 + end + + config.vm.synced_folder '.', '/vagrant', disabled: true + + config.vm.provision 'ansible' do |ansible| + ansible.config_file = '../../../ansible.cfg' + ansible.playbook = '../../../run_all.yml' + end +end diff --git a/test/ansible/vagrant/experimental/hardenedbsd-13/bootstrap b/test/ansible/vagrant/experimental/hardenedbsd-13/bootstrap new file mode 100755 index 000000000..370b1b68e --- /dev/null +++ b/test/ansible/vagrant/experimental/hardenedbsd-13/bootstrap @@ -0,0 +1,5 @@ +#!/bin/sh +unset IFS +set -euf + +sudo pkg install python3 diff --git a/test/ansible/vagrant/experimental/openbsd-6/Vagrantfile b/test/ansible/vagrant/experimental/openbsd-6/Vagrantfile deleted file mode 100644 index 1a1fc3421..000000000 --- a/test/ansible/vagrant/experimental/openbsd-6/Vagrantfile +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -Vagrant.configure('2') do |config| - config.vm.box = 'generic/openbsd6' - # config.vm.box_version = '4.2.16' - config.vm.provision "shell", inline: <<-SHELL - sudo pkg_add python py3-pip gcc openssl-1.0.2up3 gtar-1.34 - # sudo pkg_add -u - # sudo pkg_add kitty - SHELL -end - -common = '../../common' -load common if File.exist?(common) diff --git a/test/ansible/vagrant/experimental/openbsd-6/skip b/test/ansible/vagrant/experimental/openbsd-6/skip deleted file mode 100755 index 18d4c60f4..000000000 --- a/test/ansible/vagrant/experimental/openbsd-6/skip +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -die() { - echo "$@" >&2 - exit 1 -} - -[ "${PACKAGE_TESTING}" = "true" ] && die "no package available for this distribution" -exit 0 diff --git a/test/ansible/vagrant/experimental/openbsd-7/Vagrantfile b/test/ansible/vagrant/experimental/openbsd-7/Vagrantfile index 1c11a94dc..d7f9801e1 100644 --- a/test/ansible/vagrant/experimental/openbsd-7/Vagrantfile +++ b/test/ansible/vagrant/experimental/openbsd-7/Vagrantfile @@ -2,13 +2,19 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/openbsd7' - # config.vm.box_version = '4.2.16' - config.vm.provision "shell", inline: <<-SHELL - sudo pkg_add python-3.9.16 py3-pip gcc-11.2.0p3 openssl-3.0.8 gtar-1.34 - # sudo pkg_add -u - # sudo pkg_add kitty - SHELL -end + config.vm.define 'crowdsec' -common = '../../common' -load common if File.exist?(common) + config.vm.provision 'shell', path: 'bootstrap' + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 1 + libvirt.memory = 1536 + end + + config.vm.synced_folder '.', '/vagrant', disabled: true + + config.vm.provision 'ansible' do |ansible| + ansible.config_file = '../../../ansible.cfg' + ansible.playbook = '../../../run_all.yml' + end +end diff --git a/test/ansible/vagrant/experimental/openbsd-7/bootstrap b/test/ansible/vagrant/experimental/openbsd-7/bootstrap new file mode 100755 index 000000000..3b2480d3b --- /dev/null +++ b/test/ansible/vagrant/experimental/openbsd-7/bootstrap @@ -0,0 +1,6 @@ +#!/bin/sh +unset IFS +set -euf + +sudo pkg_add -u +sudo pkg_add python-3.9.13 py3-pip gcc-11.2.0p2 openssl-3.0.3p0 gtar-1.34 truncate-5.2.1 diff --git a/test/ansible/vagrant/fedora-33/Vagrantfile b/test/ansible/vagrant/fedora-33/Vagrantfile index df6f06944..49b5ee990 100644 --- a/test/ansible/vagrant/fedora-33/Vagrantfile +++ b/test/ansible/vagrant/fedora-33/Vagrantfile @@ -3,8 +3,6 @@ Vagrant.configure('2') do |config| # config.vm.box = "fedora/33-cloud-base" config.vm.box = 'generic/fedora33' - config.vm.provision "shell", inline: <<-SHELL - SHELL end common = '../common' diff --git a/test/ansible/vagrant/fedora-34/Vagrantfile b/test/ansible/vagrant/fedora-34/Vagrantfile index db2db8d08..1d172c9c7 100644 --- a/test/ansible/vagrant/fedora-34/Vagrantfile +++ b/test/ansible/vagrant/fedora-34/Vagrantfile @@ -3,8 +3,6 @@ Vagrant.configure('2') do |config| # config.vm.box = "fedora/34-cloud-base" config.vm.box = 'generic/fedora34' - config.vm.provision "shell", inline: <<-SHELL - SHELL end common = '../common' diff --git a/test/ansible/vagrant/fedora-35/Vagrantfile b/test/ansible/vagrant/fedora-35/Vagrantfile index c52604340..f11730764 100644 --- a/test/ansible/vagrant/fedora-35/Vagrantfile +++ b/test/ansible/vagrant/fedora-35/Vagrantfile @@ -3,8 +3,6 @@ Vagrant.configure('2') do |config| # config.vm.box = 'fedora/35-cloud-base' config.vm.box = 'generic/fedora35' - config.vm.provision "shell", inline: <<-SHELL - SHELL end common = '../common' diff --git a/test/ansible/vagrant/fedora-36/Vagrantfile b/test/ansible/vagrant/fedora-36/Vagrantfile index 0cd4fafd0..ef80f514c 100644 --- a/test/ansible/vagrant/fedora-36/Vagrantfile +++ b/test/ansible/vagrant/fedora-36/Vagrantfile @@ -3,8 +3,6 @@ Vagrant.configure('2') do |config| # config.vm.box = "fedora/36-cloud-base" config.vm.box = 'generic/fedora36' - config.vm.provision "shell", inline: <<-SHELL - SHELL end common = '../common' diff --git a/test/ansible/vagrant/fedora-37/Vagrantfile b/test/ansible/vagrant/fedora-37/Vagrantfile deleted file mode 100644 index 0eb8978e3..000000000 --- a/test/ansible/vagrant/fedora-37/Vagrantfile +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -Vagrant.configure('2') do |config| - config.vm.box = 'generic/fedora37' - config.vm.provision "shell", inline: <<-SHELL - sudo dnf -y install kitty-terminfo - SHELL -end - -common = '../common' -load common if File.exist?(common) diff --git a/test/ansible/vagrant/fedora-38/Vagrantfile b/test/ansible/vagrant/fedora-38/Vagrantfile deleted file mode 100644 index 0e5bf79ed..000000000 --- a/test/ansible/vagrant/fedora-38/Vagrantfile +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -Vagrant.configure('2') do |config| - config.vm.box = "fedora/38-cloud-base" - config.vm.provision "shell", inline: <<-SHELL - SHELL -end - -common = '../common' -load common if File.exist?(common) diff --git a/test/ansible/vagrant/freebsd-12/Vagrantfile b/test/ansible/vagrant/freebsd-12/Vagrantfile index b500a64d9..33e6b473f 100644 --- a/test/ansible/vagrant/freebsd-12/Vagrantfile +++ b/test/ansible/vagrant/freebsd-12/Vagrantfile @@ -2,9 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/freebsd12' - config.vm.provision "shell", inline: <<-SHELL - pkg install -y gtar - SHELL end common = '../common' diff --git a/test/ansible/vagrant/freebsd-13/Vagrantfile b/test/ansible/vagrant/freebsd-13/Vagrantfile index f416ad677..851c04254 100644 --- a/test/ansible/vagrant/freebsd-13/Vagrantfile +++ b/test/ansible/vagrant/freebsd-13/Vagrantfile @@ -2,9 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/freebsd13' - config.vm.provision "shell", inline: <<-SHELL - pkg install -y gtar - SHELL end common = '../common' diff --git a/test/ansible/vagrant/oracle-7/Vagrantfile b/test/ansible/vagrant/oracle-7/Vagrantfile index bd435ca54..638a6123b 100644 --- a/test/ansible/vagrant/oracle-7/Vagrantfile +++ b/test/ansible/vagrant/oracle-7/Vagrantfile @@ -2,9 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/oracle7' - config.vm.provision "shell", inline: <<-SHELL - sudo yum-config-manager --enable ol7_optional_latest - SHELL end common = '../common' diff --git a/test/ansible/vagrant/oracle-8/Vagrantfile b/test/ansible/vagrant/oracle-8/Vagrantfile index 6744ea87d..425ad5e16 100644 --- a/test/ansible/vagrant/oracle-8/Vagrantfile +++ b/test/ansible/vagrant/oracle-8/Vagrantfile @@ -2,9 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/oracle8' - config.vm.provision "shell", inline: <<-SHELL - sudo dnf config-manager --set-enabled ol8_codeready_builder - SHELL end common = '../common' diff --git a/test/ansible/vagrant/oracle-9/Vagrantfile b/test/ansible/vagrant/oracle-9/Vagrantfile index 8dcb4a13b..d4e3f618f 100644 --- a/test/ansible/vagrant/oracle-9/Vagrantfile +++ b/test/ansible/vagrant/oracle-9/Vagrantfile @@ -2,9 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/oracle9' - config.vm.provision "shell", inline: <<-SHELL - sudo dnf config-manager --set-enabled ol9_codeready_builder - SHELL end common = '../common' diff --git a/test/ansible/vagrant/rocky-8/Vagrantfile b/test/ansible/vagrant/rocky-8/Vagrantfile index 27caf80c7..c7315cc67 100644 --- a/test/ansible/vagrant/rocky-8/Vagrantfile +++ b/test/ansible/vagrant/rocky-8/Vagrantfile @@ -2,10 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/rocky8' - config.vm.provision "shell", inline: <<-SHELL - sudo dnf config-manager --set-enabled powertools - sudo dnf -y install kitty-terminfo - SHELL end common = '../common' diff --git a/test/ansible/vagrant/rocky-9/Vagrantfile b/test/ansible/vagrant/rocky-9/Vagrantfile index a4d06e4a9..0adb3ab51 100644 --- a/test/ansible/vagrant/rocky-9/Vagrantfile +++ b/test/ansible/vagrant/rocky-9/Vagrantfile @@ -2,10 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/rocky9' - config.vm.provision "shell", inline: <<-SHELL - sudo dnf config-manager --set-enabled crb - sudo dnf -y install kitty-terminfo - SHELL end common = '../common' diff --git a/test/ansible/vagrant/ubuntu-16.04-xenial/Vagrantfile b/test/ansible/vagrant/ubuntu-16.04-xenial/Vagrantfile index 3af880994..86646ee7a 100644 --- a/test/ansible/vagrant/ubuntu-16.04-xenial/Vagrantfile +++ b/test/ansible/vagrant/ubuntu-16.04-xenial/Vagrantfile @@ -2,8 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/ubuntu1604' - config.vm.provision "shell", inline: <<-SHELL - SHELL end common = '../common' diff --git a/test/ansible/vagrant/ubuntu-18.04-bionic/Vagrantfile b/test/ansible/vagrant/ubuntu-18.04-bionic/Vagrantfile index 5e25a2ee8..70a77806b 100644 --- a/test/ansible/vagrant/ubuntu-18.04-bionic/Vagrantfile +++ b/test/ansible/vagrant/ubuntu-18.04-bionic/Vagrantfile @@ -3,8 +3,6 @@ Vagrant.configure('2') do |config| # the official boxes only supports virtualbox config.vm.box = 'generic/ubuntu1804' - config.vm.provision "shell", inline: <<-SHELL - SHELL end common = '../common' diff --git a/test/ansible/vagrant/ubuntu-20.04-focal/Vagrantfile b/test/ansible/vagrant/ubuntu-20.04-focal/Vagrantfile index ea5b33907..0006ae926 100644 --- a/test/ansible/vagrant/ubuntu-20.04-focal/Vagrantfile +++ b/test/ansible/vagrant/ubuntu-20.04-focal/Vagrantfile @@ -2,9 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/ubuntu2004' - config.vm.provision "shell", inline: <<-SHELL - sudo apt install -y kitty-terminfo - SHELL end common = '../common' diff --git a/test/ansible/vagrant/ubuntu-22.04-jammy/Vagrantfile b/test/ansible/vagrant/ubuntu-22.04-jammy/Vagrantfile index 9e17f71fb..c0ccee54b 100644 --- a/test/ansible/vagrant/ubuntu-22.04-jammy/Vagrantfile +++ b/test/ansible/vagrant/ubuntu-22.04-jammy/Vagrantfile @@ -2,8 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/ubuntu2204' - config.vm.provision "shell", inline: <<-SHELL - SHELL end common = '../common' diff --git a/test/ansible/vagrant/ubuntu-22.10-kinetic/Vagrantfile b/test/ansible/vagrant/ubuntu-22.10-kinetic/Vagrantfile deleted file mode 100644 index 6c15b0a1e..000000000 --- a/test/ansible/vagrant/ubuntu-22.10-kinetic/Vagrantfile +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -Vagrant.configure('2') do |config| - config.vm.box = 'generic/ubuntu2210' - config.vm.provision "shell", inline: <<-SHELL - SHELL -end - -common = '../common' -load common if File.exist?(common) diff --git a/test/ansible/vagrant/ubuntu-23.04-lunar/Vagrantfile b/test/ansible/vagrant/ubuntu-23.04-lunar/Vagrantfile deleted file mode 100644 index f40fb7bd5..000000000 --- a/test/ansible/vagrant/ubuntu-23.04-lunar/Vagrantfile +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -Vagrant.configure('2') do |config| - config.vm.box = 'bento/ubuntu-23.04' - config.vm.provision "shell", inline: <<-SHELL - SHELL -end - -common = '../common' -load common if File.exist?(common) diff --git a/test/ansible/vagrant/wizard/centos-8/Vagrantfile b/test/ansible/vagrant/wizard/centos-8/Vagrantfile index 9db09a4ce..2df31a392 100644 --- a/test/ansible/vagrant/wizard/centos-8/Vagrantfile +++ b/test/ansible/vagrant/wizard/centos-8/Vagrantfile @@ -1,13 +1,38 @@ # frozen_string_literal: true Vagrant.configure('2') do |config| - config.vm.box = 'generic/centos8s' - config.vm.provision "shell", inline: <<-SHELL - sudo dnf -y install dnf-plugins-core kitty-terminfo - dnf config-manager --set-enabled powertools - # sudo dnf -y update - SHELL -end + config.vm.box = 'centos/stream8' + config.vm.define 'wizard' -common = '../common' -load common if File.exists?(common) + config.vm.provision 'shell', path: 'bootstrap' + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 4 + libvirt.memory = 4096 + end + + config.vm.synced_folder '.', '/vagrant', disabled: true + + # install the dependencies for functional tests + + config.vm.provision 'ansible' do |provdep| + provdep.config_file = '../../../ansible.cfg' + provdep.playbook = '../../../provision_dependencies.yml' + end + + config.vm.provision 'ansible' do |provtest| + provtest.config_file = '../../../ansible.cfg' + provtest.playbook = '../../../provision_test_suite.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../prepare_tests.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../run_wizard_tests.yml' + end + +end diff --git a/test/ansible/vagrant/wizard/centos-8/bootstrap b/test/ansible/vagrant/wizard/centos-8/bootstrap new file mode 100755 index 000000000..b33ad9c88 --- /dev/null +++ b/test/ansible/vagrant/wizard/centos-8/bootstrap @@ -0,0 +1,5 @@ +#!/bin/sh +unset IFS +set -euf + +sudo dnf -y update diff --git a/test/ansible/vagrant/wizard/common b/test/ansible/vagrant/wizard/common deleted file mode 100644 index be1820914..000000000 --- a/test/ansible/vagrant/wizard/common +++ /dev/null @@ -1,67 +0,0 @@ -# vim: set ft=ruby: -# frozen_string_literal: true - -def find_ansible_cfg - path = Pathname.new(Dir.pwd) - until path.root? - ansible_cfg = path + 'ansible.cfg' - return path if ansible_cfg.exist? - path = path.parent - end - nil # return nil if not found -end - -Vagrant.configure('2') do |config| - config.vm.define 'wizard' - - if ARGV.any? { |arg| arg == 'up' || arg == 'provision' } - unless ENV['DB_BACKEND'] - $stderr.puts "\e[31mThe DB_BACKEND environment variable is not defined. Please set up the environment and try again.\e[0m" - exit 1 - end - end - - config.vm.provision 'shell', path: 'bootstrap' if File.exists?('bootstrap') - config.vm.synced_folder '.', '/vagrant', disabled: true - - config.vm.provider :libvirt do |libvirt| - libvirt.cpus = 4 - libvirt.memory = 4096 - end - - path = find_ansible_cfg - if !path - puts "ansible.cfg not found" - end - - config.vm.provision 'ansible' do |ansible| - ansible.config_file = (path + 'ansible.cfg').to_s - ansible.playbook = (path + 'provision_dependencies.yml').to_s - ansible.compatibility_mode = "2.0" - end - - config.vm.provision 'ansible' do |ansible| - ansible.config_file = (path + 'ansible.cfg').to_s - ansible.playbook = (path + 'provision_test_suite.yml').to_s - ansible.compatibility_mode = "2.0" - end - - config.vm.provision 'ansible' do |ansible| - ansible.config_file = (path + 'ansible.cfg').to_s - ansible.playbook = (path + 'prepare_tests.yml').to_s - ansible.compatibility_mode = "2.0" - end - - config.vm.provision 'ansible' do |ansible| - ansible.config_file = (path + 'ansible.cfg').to_s - ansible.playbook = (path + 'debug_tools.yml').to_s - ansible.compatibility_mode = "2.0" - end - - config.vm.provision 'ansible' do |ansible| - ansible.config_file = (path + 'ansible.cfg').to_s - ansible.playbook = (path + 'run_wizard_tests.yml').to_s - ansible.compatibility_mode = "2.0" - end - -end diff --git a/test/ansible/vagrant/wizard/debian-10-buster/Vagrantfile b/test/ansible/vagrant/wizard/debian-10-buster/Vagrantfile deleted file mode 100644 index 3b10b312d..000000000 --- a/test/ansible/vagrant/wizard/debian-10-buster/Vagrantfile +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -Vagrant.configure('2') do |config| - config.vm.box = 'debian/buster64' - config.vm.provision "shell", inline: <<-SHELL - sudo apt update - sudo apt install -y aptitude kitty-terminfo - SHELL -end - -common = '../common' -load common if File.exists?(common) diff --git a/test/ansible/vagrant/wizard/debian-11-bullseye/Vagrantfile b/test/ansible/vagrant/wizard/debian-11-bullseye/Vagrantfile deleted file mode 100644 index 6dd7bb2fc..000000000 --- a/test/ansible/vagrant/wizard/debian-11-bullseye/Vagrantfile +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -Vagrant.configure('2') do |config| - config.vm.box = 'debian/bullseye64' - config.vm.provision "shell", inline: <<-SHELL - sudo apt update - sudo apt install -y aptitude kitty-terminfo - SHELL -end - -common = '../common' -load common if File.exists?(common) diff --git a/test/ansible/vagrant/wizard/debian-12-bookworm/Vagrantfile b/test/ansible/vagrant/wizard/debian-12-bookworm/Vagrantfile deleted file mode 100644 index 5ccf234eb..000000000 --- a/test/ansible/vagrant/wizard/debian-12-bookworm/Vagrantfile +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -Vagrant.configure('2') do |config| - config.vm.box = 'debian/bookworm64' - config.vm.provision "shell", inline: <<-SHELL - sudo apt update - sudo apt install -y aptitude kitty-terminfo - SHELL -end - -common = '../common' -load common if File.exists?(common) diff --git a/test/ansible/vagrant/wizard/debian-bullseye/Vagrantfile b/test/ansible/vagrant/wizard/debian-bullseye/Vagrantfile new file mode 100644 index 000000000..1d6993a4e --- /dev/null +++ b/test/ansible/vagrant/wizard/debian-bullseye/Vagrantfile @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'debian/bullseye64' + config.vm.define 'wizard' + + config.vm.provision 'shell', path: 'bootstrap' + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 4 + libvirt.memory = 4096 + end + + config.vm.synced_folder '.', '/vagrant', disabled: true + + # install the dependencies for functional tests + + config.vm.provision 'ansible' do |provdep| + provdep.config_file = '../../../ansible.cfg' + provdep.playbook = '../../../provision_dependencies.yml' + end + + config.vm.provision 'ansible' do |provtest| + provtest.config_file = '../../../ansible.cfg' + provtest.playbook = '../../../provision_test_suite.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../prepare_tests.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../debug_tools.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../run_wizard_tests.yml' + end + +end diff --git a/test/ansible/vagrant/wizard/debian-bullseye/bootstrap b/test/ansible/vagrant/wizard/debian-bullseye/bootstrap new file mode 100755 index 000000000..6a5df521a --- /dev/null +++ b/test/ansible/vagrant/wizard/debian-bullseye/bootstrap @@ -0,0 +1,5 @@ +#!/bin/sh +unset IFS +set -euf + +sudo apt install -y aptitude diff --git a/test/ansible/vagrant/wizard/debian-buster/Vagrantfile b/test/ansible/vagrant/wizard/debian-buster/Vagrantfile new file mode 100644 index 000000000..d4380a945 --- /dev/null +++ b/test/ansible/vagrant/wizard/debian-buster/Vagrantfile @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'debian/buster64' + config.vm.define 'wizard' + + config.vm.provision 'shell', path: 'bootstrap' + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 4 + libvirt.memory = 4096 + end + + config.vm.synced_folder '.', '/vagrant', disabled: true + + # install the dependencies for functional tests + + config.vm.provision 'ansible' do |provdep| + provdep.config_file = '../../../ansible.cfg' + provdep.playbook = '../../../provision_dependencies.yml' + end + + config.vm.provision 'ansible' do |provtest| + provtest.config_file = '../../../ansible.cfg' + provtest.playbook = '../../../provision_test_suite.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../prepare_tests.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../debug_tools.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../run_wizard_tests.yml' + end + +end diff --git a/test/ansible/vagrant/wizard/debian-buster/bootstrap b/test/ansible/vagrant/wizard/debian-buster/bootstrap new file mode 100755 index 000000000..6a5df521a --- /dev/null +++ b/test/ansible/vagrant/wizard/debian-buster/bootstrap @@ -0,0 +1,5 @@ +#!/bin/sh +unset IFS +set -euf + +sudo apt install -y aptitude diff --git a/test/ansible/vagrant/wizard/fedora-36/Vagrantfile b/test/ansible/vagrant/wizard/fedora-36/Vagrantfile index 969a8e70c..09ee8a39d 100644 --- a/test/ansible/vagrant/wizard/fedora-36/Vagrantfile +++ b/test/ansible/vagrant/wizard/fedora-36/Vagrantfile @@ -2,10 +2,37 @@ Vagrant.configure('2') do |config| config.vm.box = 'fedora/36-cloud-base' - config.vm.provision "shell", inline: <<-SHELL - # sudo dnf -y update - SHELL -end + config.vm.define 'wizard' -common = '../common' -load common if File.exists?(common) + config.vm.provision 'shell', path: 'bootstrap' + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 4 + libvirt.memory = 4096 + end + + config.vm.synced_folder '.', '/vagrant', disabled: true + + # install the dependencies for functional tests + + config.vm.provision 'ansible' do |provdep| + provdep.config_file = '../../../ansible.cfg' + provdep.playbook = '../../../provision_dependencies.yml' + end + + config.vm.provision 'ansible' do |provtest| + provtest.config_file = '../../../ansible.cfg' + provtest.playbook = '../../../provision_test_suite.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../prepare_tests.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../run_wizard_tests.yml' + end + +end diff --git a/test/ansible/vagrant/wizard/fedora-36/bootstrap b/test/ansible/vagrant/wizard/fedora-36/bootstrap new file mode 100755 index 000000000..b33ad9c88 --- /dev/null +++ b/test/ansible/vagrant/wizard/fedora-36/bootstrap @@ -0,0 +1,5 @@ +#!/bin/sh +unset IFS +set -euf + +sudo dnf -y update diff --git a/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/Vagrantfile b/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/Vagrantfile index c13d2f946..933dabab5 100644 --- a/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/Vagrantfile +++ b/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/Vagrantfile @@ -2,10 +2,42 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/ubuntu2204' - config.vm.provision "shell", inline: <<-SHELL - sudo apt install -y aptitude kitty-terminfo - SHELL -end + config.vm.define 'wizard' -common = '../common' -load common if File.exists?(common) + config.vm.provision 'shell', path: 'bootstrap' + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 4 + libvirt.memory = 4096 + end + + config.vm.synced_folder '.', '/vagrant', disabled: true + + # install the dependencies for functional tests + + config.vm.provision 'ansible' do |provdep| + provdep.config_file = '../../../ansible.cfg' + provdep.playbook = '../../../provision_dependencies.yml' + end + + config.vm.provision 'ansible' do |provtest| + provtest.config_file = '../../../ansible.cfg' + provtest.playbook = '../../../provision_test_suite.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../prepare_tests.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../debug_tools.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../run_wizard_tests.yml' + end + +end diff --git a/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/bootstrap b/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/bootstrap new file mode 100755 index 000000000..6a5df521a --- /dev/null +++ b/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/bootstrap @@ -0,0 +1,5 @@ +#!/bin/sh +unset IFS +set -euf + +sudo apt install -y aptitude diff --git a/test/ansible/vagrant/wizard/ubuntu-22.10-kinetic/Vagrantfile b/test/ansible/vagrant/wizard/ubuntu-22.10-kinetic/Vagrantfile index d0e2e3cda..c3e8759aa 100644 --- a/test/ansible/vagrant/wizard/ubuntu-22.10-kinetic/Vagrantfile +++ b/test/ansible/vagrant/wizard/ubuntu-22.10-kinetic/Vagrantfile @@ -2,10 +2,42 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/ubuntu2210' - config.vm.provision "shell", inline: <<-SHELL - sudo apt install -y aptitude kitty-terminfo - SHELL -end + config.vm.define 'wizard' -common = '../common' -load common if File.exists?(common) + config.vm.provision 'shell', path: 'bootstrap' + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 4 + libvirt.memory = 4096 + end + + config.vm.synced_folder '.', '/vagrant', disabled: true + + # install the dependencies for functional tests + + config.vm.provision 'ansible' do |provdep| + provdep.config_file = '../../../ansible.cfg' + provdep.playbook = '../../../provision_dependencies.yml' + end + + config.vm.provision 'ansible' do |provtest| + provtest.config_file = '../../../ansible.cfg' + provtest.playbook = '../../../provision_test_suite.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../prepare_tests.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../debug_tools.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../run_wizard_tests.yml' + end + +end diff --git a/test/ansible/vars/go.yml b/test/ansible/vars/go.yml index 929e412cd..683b4fbbe 100644 --- a/test/ansible/vars/go.yml +++ b/test/ansible/vars/go.yml @@ -1,5 +1,5 @@ # vim: set ft=yaml.ansible: --- -golang_version: "1.20.5" +golang_version: "1.20.4" golang_install_dir: "/opt/go/{{ golang_version }}" diff --git a/test/bats-detect/openresty-deb.bats b/test/bats-detect/openresty-deb.bats index 0c8bc3c9a..c1e91949d 100644 --- a/test/bats-detect/openresty-deb.bats +++ b/test/bats-detect/openresty-deb.bats @@ -37,13 +37,8 @@ setup() { run -0 sudo gpg --yes --dearmor -o /usr/share/keyrings/openresty.gpg < <(output) run -0 sudo tee <<< "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/openresty.gpg] http://openresty.org/package/ubuntu $(lsb_release -sc) main" /etc/apt/sources.list.d/openresty.list else - release="$(lsb_release -sc)" - # Debian 12 package is not available as of 2023-07-3 - if [[ "$release" == "bookworm" ]]; then - release="bullseye" - fi run -0 sudo apt-key add - < <(output) - run -0 sudo tee <<< "deb http://openresty.org/package/debian $release openresty" /etc/apt/sources.list.d/openresty.list + run -0 sudo tee <<< "deb http://openresty.org/package/debian $(lsb_release -sc) openresty" /etc/apt/sources.list.d/openresty.list fi run -0 deb-update run -0 deb-install openresty diff --git a/test/bats-detect/openresty-rpm.bats b/test/bats-detect/openresty-rpm.bats index d4c3661bc..6fc0a8a09 100644 --- a/test/bats-detect/openresty-rpm.bats +++ b/test/bats-detect/openresty-rpm.bats @@ -34,14 +34,10 @@ setup() { run -0 rpm-install redhat-lsb-core if [[ "$(lsb_release -is)" == "Fedora" ]]; then run -0 sudo curl -1sSLf "https://openresty.org/package/fedora/openresty.repo" -o "/etc/yum.repos.d/openresty.repo" - elif [[ "$(lsb_release -is)" == CentOS* ]]; then # must match CentOSStream + elif [[ "$(lsb_release -is)" == "CentOS" ]]; then run -0 sudo curl -1sSLf "https://openresty.org/package/centos/openresty.repo" -o "/etc/yum.repos.d/openresty.repo" fi - run sudo dnf check-update - # 0 = up to date, 100 = updates available - if [[ "$status" -ne 0 ]] && [[ "$status" -ne 100 ]]; then - fail "dnf check-update failed with status $status" - fi + run -0 sudo dnf check-update run -0 rpm-install openresty run -0 sudo systemctl enable openresty.service } diff --git a/test/bats.mk b/test/bats.mk index 4eb7abcbf..65bb4a286 100644 --- a/test/bats.mk +++ b/test/bats.mk @@ -10,7 +10,7 @@ ifdef PACKAGE_TESTING INIT_BACKEND = systemd CONFIG_BACKEND = global else - # LOCAL_DIR contains a local instance of crowdsec, complete with + # LOCAL_DIR will contain contains a local instance of crowdsec, complete with # configuration and data LOCAL_DIR = $(TEST_DIR)/local BIN_DIR = $(LOCAL_DIR)/bin @@ -51,7 +51,6 @@ export CONFIG_BACKEND="$(CONFIG_BACKEND)" export PACKAGE_TESTING="$(PACKAGE_TESTING)" export TEST_COVERAGE="$(TEST_COVERAGE)" export GOCOVERDIR="$(TEST_DIR)/coverage" -export PATH="$(TEST_DIR)/tools:$(PATH)" endef bats-all: bats-clean bats-build bats-fixture bats-test bats-test-hub diff --git a/test/bats/01_crowdsec.bats b/test/bats/01_crowdsec.bats index a1a2861f6..75b29033e 100644 --- a/test/bats/01_crowdsec.bats +++ b/test/bats/01_crowdsec.bats @@ -151,10 +151,9 @@ teardown() { rm -f "$ACQUIS_DIR" config_set '.common.log_media="stdout"' - rune -1 timeout 2s "${CROWDSEC}" + rune -124 timeout 2s "${CROWDSEC}" # check warning - assert_stderr --partial "no acquisition file found" - assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" + assert_stderr_line --partial "no acquisition file found" } @test "crowdsec (error if acquisition_path and acquisition_dir are not defined)" { @@ -167,56 +166,19 @@ teardown() { config_set '.crowdsec_service.acquisition_dir=""' config_set '.common.log_media="stdout"' - rune -1 timeout 2s "${CROWDSEC}" + rune -124 timeout 2s "${CROWDSEC}" # check warning - assert_stderr --partial "no acquisition_path or acquisition_dir specified" - assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" + assert_stderr_line --partial "no acquisition_path or acquisition_dir specified" } @test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" { - ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path') - config_set '.crowdsec_service.acquisition_path=""' - - ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir') - mkdir -p "$ACQUIS_DIR" - mv "$ACQUIS_YAML" "$ACQUIS_DIR"/foo.yaml - - rune -124 timeout 2s "${CROWDSEC}" - - # now, if foo.yaml is empty instead, there won't be valid datasources. - - cat /dev/null >"$ACQUIS_DIR"/foo.yaml - - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" -} - -@test "crowdsec (disabled datasources)" { - config_set '.common.log_media="stdout"' - - # a datasource cannot run - missing journalctl command - - ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir') - mkdir -p "$ACQUIS_DIR" - cat >"$ACQUIS_DIR"/foo.yaml <<-EOT - source: journalctl - journalctl_filter: - - "_SYSTEMD_UNIT=ssh.service" - labels: - type: syslog - EOT - - rune -124 timeout 2s env PATH='' "${CROWDSEC}" - #shellcheck disable=SC2016 - assert_stderr --partial 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in $PATH' - - # if all datasources are disabled, crowdsec should exit - ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path') rm -f "$ACQUIS_YAML" config_set '.crowdsec_service.acquisition_path=""' - rune -1 timeout 2s env PATH='' "${CROWDSEC}" - assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" -} + ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir') + mkdir -p "$ACQUIS_DIR" + touch "$ACQUIS_DIR"/foo.yaml + rune -124 timeout 2s "${CROWDSEC}" +} diff --git a/test/bats/01_cscli.bats b/test/bats/01_cscli.bats index ef825d2a8..a01d936b7 100644 --- a/test/bats/01_cscli.bats +++ b/test/bats/01_cscli.bats @@ -101,9 +101,9 @@ teardown() { # check that LAPI configuration is loaded (human and json, not shows in raw) rune -0 cscli config show -o human - assert_line --regexp ".*- URL +: http://127.0.0.1:8080/" - assert_line --regexp ".*- Login +: githubciXXXXXXXXXXXXXXXXXXXXXXXX" - assert_line --regexp ".*- Credentials File +: .*/local_api_credentials.yaml" + assert_line --regexp ".*- URL\s+: http://127.0.0.1:8080/" + assert_line --regexp ".*- Login\s+: githubciXXXXXXXXXXXXXXXXXXXXXXXX" + assert_line --regexp ".*- Credentials File\s+: .*/local_api_credentials.yaml" rune -0 cscli config show -o json rune -0 jq -c '.API.Client.Credentials | [.url,.login]' <(output) @@ -182,7 +182,7 @@ teardown() { @test "cscli - empty LAPI credentials file" { LOCAL_API_CREDENTIALS=$(config_get '.api.client.credentials_path') - : > "${LOCAL_API_CREDENTIALS}" + truncate -s 0 "${LOCAL_API_CREDENTIALS}" rune -1 cscli lapi status assert_stderr --partial "no credentials or URL found in api client configuration '${LOCAL_API_CREDENTIALS}'" diff --git a/test/bats/04_capi.bats b/test/bats/04_capi.bats index f4c9f49e0..7015f2c5d 100644 --- a/test/bats/04_capi.bats +++ b/test/bats/04_capi.bats @@ -60,5 +60,5 @@ setup() { ONLINE_API_CREDENTIALS_YAML="$(config_get '.api.server.online_client.credentials_path')" rm "${ONLINE_API_CREDENTIALS_YAML}" rune -1 cscli capi status - assert_stderr --partial "local API is disabled, please run this command on the local API machine: loading online client credentials: failed to read api server credentials configuration file '${ONLINE_API_CREDENTIALS_YAML}': open ${ONLINE_API_CREDENTIALS_YAML}: no such file or directory" + assert_stderr --partial "Local API is disabled, please run this command on the local API machine: loading online client credentials: failed to read api server credentials configuration file '${ONLINE_API_CREDENTIALS_YAML}': open ${ONLINE_API_CREDENTIALS_YAML}: no such file or directory" } diff --git a/test/bats/72_plugin_badconfig.bats b/test/bats/72_plugin_badconfig.bats index 9640e3330..c216067f9 100644 --- a/test/bats/72_plugin_badconfig.bats +++ b/test/bats/72_plugin_badconfig.bats @@ -36,35 +36,35 @@ teardown() { config_set '.plugin_config.user="" | .plugin_config.group="nogroup"' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: both plugin user and group must be set" + assert_stderr --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: both plugin user and group must be set" } @test "misconfigured plugin, only group is empty" { config_set '(.plugin_config.user="nobody") | (.plugin_config.group="")' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: both plugin user and group must be set" + assert_stderr --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: both plugin user and group must be set" } @test "misconfigured plugin, user does not exist" { config_set '(.plugin_config.user="userdoesnotexist") | (.plugin_config.group="groupdoesnotexist")' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: user: unknown user userdoesnotexist" + assert_stderr --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: user: unknown user userdoesnotexist" } @test "misconfigured plugin, group does not exist" { config_set '(.plugin_config.user=strenv(USER)) | (.plugin_config.group="groupdoesnotexist")' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: group: unknown group groupdoesnotexist" + assert_stderr --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: group: unknown group groupdoesnotexist" } @test "bad plugin name" { config_set "${PROFILES_PATH}" '.notifications=["http_default"]' cp "${PLUGIN_DIR}"/notification-http "${PLUGIN_DIR}"/badname rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: plugin name ${PLUGIN_DIR}/badname is invalid. Name should be like {type-name}" + assert_stderr --partial "api server init: unable to run local API: while loading plugin: plugin name ${PLUGIN_DIR}/badname is invalid. Name should be like {type-name}" } @test "duplicate notification config" { @@ -85,14 +85,14 @@ teardown() { config_set "${PROFILES_PATH}" '.notifications=["http_default"]' chmod g+w "${PLUGIN_DIR}"/notification-http rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is group writable, group writable plugins are invalid" + assert_stderr --partial "api server init: unable to run local API: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is group writable, group writable plugins are invalid" } @test "bad plugin permission (world writable)" { config_set "${PROFILES_PATH}" '.notifications=["http_default"]' chmod o+w "${PLUGIN_DIR}"/notification-http rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is world writable, world writable plugins are invalid" + assert_stderr --partial "api server init: unable to run local API: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is world writable, world writable plugins are invalid" } @test "config.yaml: missing .plugin_config section" { @@ -116,9 +116,9 @@ teardown() { assert_stderr --partial "api server init: plugins are enabled, but config_paths.plugin_dir is not defined" } -@test "unable to run plugin broker: while reading plugin config" { +@test "unable to run local API: while reading plugin config" { config_set '.config_paths.notification_dir="/this/path/does/not/exist"' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin config: open /this/path/does/not/exist: no such file or directory" + assert_stderr --partial "api server init: unable to run local API: while loading plugin config: open /this/path/does/not/exist: no such file or directory" } diff --git a/test/bats/90_decisions.bats b/test/bats/90_decisions.bats index bcb410de9..3499f3e0e 100644 --- a/test/bats/90_decisions.bats +++ b/test/bats/90_decisions.bats @@ -5,9 +5,6 @@ set -u setup_file() { load "../lib/setup_file.sh" - - TESTDATA="${BATS_TEST_DIRNAME}/testdata/90_decisions" - export TESTDATA } teardown_file() { @@ -59,122 +56,8 @@ teardown() { @test "cscli decisions list, incorrect parameters" { rune -1 cscli decisions list --until toto - assert_stderr --partial 'unable to retrieve decisions: performing request: API error: while parsing duration: time: invalid duration \"toto\"' + assert_stderr --partial 'Unable to list decisions : performing request: API error: while parsing duration: time: invalid duration \"toto\"' rune -1 cscli decisions list --until toto -o json rune -0 jq -c '[.level, .msg]' <(stderr | grep "^{") - assert_output '["fatal","unable to retrieve decisions: performing request: API error: while parsing duration: time: invalid duration \"toto\""]' -} - -@test "cscli decisions import" { - # required input - rune -1 cscli decisions import - assert_stderr --partial 'required flag(s) \"input\" not set"' - - # unsupported format - rune -1 cscli decisions import -i - <<<'value\n5.6.7.8' --format xml - assert_stderr --partial "invalid format 'xml', expected one of 'json', 'csv', 'values'" - - # invalid defaults - rune -1 cscli decisions import --duration "" -i - <<<'value\n5.6.7.8' --format csv - assert_stderr --partial "--duration cannot be empty" - rune -1 cscli decisions import --scope "" -i - <<<'value\n5.6.7.8' --format csv - assert_stderr --partial "--scope cannot be empty" - rune -1 cscli decisions import --reason "" -i - <<<'value\n5.6.7.8' --format csv - assert_stderr --partial "--reason cannot be empty" - rune -1 cscli decisions import --type "" -i - <<<'value\n5.6.7.8' --format csv - assert_stderr --partial "--type cannot be empty" - - #---------- - # JSON - #---------- - - # import from file - rune -1 cscli decisions import -i "${TESTDATA}/json_decisions" - assert_stderr --partial "unable to guess format from file extension, please provide a format with --format flag" - - rune -0 cscli decisions import -i "${TESTDATA}/decisions.json" - assert_stderr --partial "Parsing json" - assert_stderr --partial "Imported 5 decisions" - - # import from stdin - rune -1 cscli decisions import -i /dev/stdin < <(cat "${TESTDATA}/decisions.json") - assert_stderr --partial "unable to guess format from file extension, please provide a format with --format flag" - rune -0 cscli decisions import -i /dev/stdin < <(cat "${TESTDATA}/decisions.json") --format json - assert_stderr --partial "Parsing json" - assert_stderr --partial "Imported 5 decisions" - - # invalid json - rune -1 cscli decisions import -i - <<<'{"blah":"blah"}' --format json - assert_stderr --partial 'Parsing json' - assert_stderr --partial 'json: cannot unmarshal object into Go value of type []main.decisionRaw' - - # json with extra data - rune -1 cscli decisions import -i - <<<'{"values":"1.2.3.4","blah":"blah"}' --format json - assert_stderr --partial 'Parsing json' - assert_stderr --partial 'json: cannot unmarshal object into Go value of type []main.decisionRaw' - - #---------- - # CSV - #---------- - - # import from file - rune -1 cscli decisions import -i "${TESTDATA}/csv_decisions" - assert_stderr --partial "unable to guess format from file extension, please provide a format with --format flag" - - rune -0 cscli decisions import -i "${TESTDATA}/decisions.csv" - assert_stderr --partial 'Parsing csv' - assert_stderr --partial 'Imported 5 decisions' - - # import from stdin - rune -1 cscli decisions import -i /dev/stdin < <(cat "${TESTDATA}/decisions.csv") - assert_stderr --partial "unable to guess format from file extension, please provide a format with --format flag" - rune -0 cscli decisions import -i /dev/stdin < <(cat "${TESTDATA}/decisions.csv") --format csv - assert_stderr --partial "Parsing csv" - assert_stderr --partial "Imported 5 decisions" - - # invalid csv - # XXX: improve validation - rune -0 cscli decisions import -i - <<<'value\n1.2.3.4,5.6.7.8' --format csv - assert_stderr --partial 'Parsing csv' - assert_stderr --partial "Imported 0 decisions" - - #---------- - # VALUES - #---------- - - # can use '-' as stdin - rune -0 cscli decisions import -i - --format values <<-EOT - 1.2.3.4 - 1.2.3.5 - 1.2.3.6 - EOT - assert_stderr --partial 'Parsing values' - assert_stderr --partial 'Imported 3 decisions' - - rune -0 cscli decisions import -i - --format values <<-EOT - 10.2.3.4 - 10.2.3.5 - 10.2.3.6 - EOT - assert_stderr --partial 'Parsing values' - assert_stderr --partial 'Imported 3 decisions' - - rune -1 cscli decisions import -i - --format values <<-EOT - whatever - EOT - assert_stderr --partial 'Parsing values' - assert_stderr --partial 'API error: unable to create alerts: whatever: invalid ip address / range' - - #---------- - # Batch - #---------- - - rune -0 cscli decisions import -i - --format values --batch 2 --debug <<-EOT - 1.2.3.4 - 1.2.3.5 - 1.2.3.6 - EOT - assert_stderr --partial 'Processing chunk of 2 decisions' - assert_stderr --partial 'Processing chunk of 1 decisions' - assert_stderr --partial 'Imported 3 decisions' + assert_output '["fatal","Unable to list decisions : performing request: API error: while parsing duration: time: invalid duration \"toto\""]' } diff --git a/test/bats/testdata/90_decisions/csv_decisions b/test/bats/testdata/90_decisions/csv_decisions deleted file mode 100644 index 858654b63..000000000 --- a/test/bats/testdata/90_decisions/csv_decisions +++ /dev/null @@ -1,6 +0,0 @@ -origin,scope,value,reason,type,duration -cscli,ip,1.6.11.16,manual import from csv,ban,1h -cscli,ip,2.7.12.17,manual import from csv,ban,1h -cscli,ip,3.8.13.18,manual import from csv,ban,1h -cscli,ip,4.9.14.19,manual import from csv,ban,1h -cscli,ip,5.10.15.20,manual import from csv,ban,1h diff --git a/test/bats/testdata/90_decisions/decisions.csv b/test/bats/testdata/90_decisions/decisions.csv deleted file mode 100644 index 858654b63..000000000 --- a/test/bats/testdata/90_decisions/decisions.csv +++ /dev/null @@ -1,6 +0,0 @@ -origin,scope,value,reason,type,duration -cscli,ip,1.6.11.16,manual import from csv,ban,1h -cscli,ip,2.7.12.17,manual import from csv,ban,1h -cscli,ip,3.8.13.18,manual import from csv,ban,1h -cscli,ip,4.9.14.19,manual import from csv,ban,1h -cscli,ip,5.10.15.20,manual import from csv,ban,1h diff --git a/test/bats/testdata/90_decisions/decisions.json b/test/bats/testdata/90_decisions/decisions.json deleted file mode 100644 index 395458c97..000000000 --- a/test/bats/testdata/90_decisions/decisions.json +++ /dev/null @@ -1,42 +0,0 @@ -[ - { - "origin": "cscli", - "scope": "ip", - "value": "1.6.11.16", - "reason": "manual import from csv", - "type": "ban", - "duration": "1h" - }, - { - "origin": "cscli", - "scope": "ip", - "value": "2.7.12.17", - "reason": "manual import from csv", - "type": "ban", - "duration": "1h" - }, - { - "origin": "cscli", - "scope": "ip", - "value": "3.8.13.18", - "reason": "manual import from csv", - "type": "ban", - "duration": "1h" - }, - { - "origin": "cscli", - "scope": "ip", - "value": "4.9.14.19", - "reason": "manual import from csv", - "type": "ban", - "duration": "1h" - }, - { - "origin": "cscli", - "scope": "ip", - "value": "5.10.15.20", - "reason": "manual import from csv", - "type": "ban", - "duration": "1h" - } -] diff --git a/test/bats/testdata/90_decisions/json_decisions b/test/bats/testdata/90_decisions/json_decisions deleted file mode 100644 index 395458c97..000000000 --- a/test/bats/testdata/90_decisions/json_decisions +++ /dev/null @@ -1,42 +0,0 @@ -[ - { - "origin": "cscli", - "scope": "ip", - "value": "1.6.11.16", - "reason": "manual import from csv", - "type": "ban", - "duration": "1h" - }, - { - "origin": "cscli", - "scope": "ip", - "value": "2.7.12.17", - "reason": "manual import from csv", - "type": "ban", - "duration": "1h" - }, - { - "origin": "cscli", - "scope": "ip", - "value": "3.8.13.18", - "reason": "manual import from csv", - "type": "ban", - "duration": "1h" - }, - { - "origin": "cscli", - "scope": "ip", - "value": "4.9.14.19", - "reason": "manual import from csv", - "type": "ban", - "duration": "1h" - }, - { - "origin": "cscli", - "scope": "ip", - "value": "5.10.15.20", - "reason": "manual import from csv", - "type": "ban", - "duration": "1h" - } -] diff --git a/test/bin/assert-crowdsec-not-running b/test/bin/assert-crowdsec-not-running index 3171287d0..b545ebf0a 100755 --- a/test/bin/assert-crowdsec-not-running +++ b/test/bin/assert-crowdsec-not-running @@ -1,15 +1,8 @@ #!/usr/bin/env bash is_crowdsec_running() { - case $(uname) in - "Linux") - # ignore processes in containers - PIDS=$(pgrep --ns $$ -x 'crowdsec') - ;; - *) - PIDS=$(pgrep -x 'crowdsec') - ;; - esac + # ignore processes in containers + PIDS=$(pgrep --ns $$ -x 'crowdsec') } # The process can be slow, especially on CI and during test coverage. diff --git a/test/bin/check-requirements b/test/bin/check-requirements index 0563eec01..351c0a01b 100755 --- a/test/bin/check-requirements +++ b/test/bin/check-requirements @@ -54,25 +54,47 @@ check_pkill() { fi } +check_yq() { + # shellcheck disable=SC2016 + howto_install='You can install it with your favorite package manager (including snap) or with "go install github.com/mikefarah/yq/v4@latest" and add ~/go/bin to $PATH.' + if ! command -v yq >/dev/null; then + die "Missing required program 'yq'. ${howto_install}" + fi + if ! (yq --version | grep mikefarah >/dev/null); then + die "yq exists but it's not the one we need (mikefarah/yq). ${howto_install}" + fi +} + check_daemonizer() { if ! command -v daemonize >/dev/null; then die "missing required program 'daemonize' (package 'daemonize' or 'https://github.com/bmc/daemonize')" fi } -echo "Checking requirements..." +check_cfssl() { + # shellcheck disable=SC2016 + howto_install='You can install it with "go install github.com/cloudflare/cfssl/cmd/cfssl@latest" and add ~/go/bin to $PATH.' + if ! command -v cfssl >/dev/null; then + die "Missing required program 'cfssl'. ${howto_install}" + fi +} -GOBIN=${TEST_DIR}/tools -export GOBIN -go install github.com/mikefarah/yq/v4@latest -go install github.com/cloudflare/cfssl/cmd/cfssl@master -go install github.com/cloudflare/cfssl/cmd/cfssljson@master +check_cfssljson() { + # shellcheck disable=SC2016 + howto_install='You can install it with "go install github.com/cloudflare/cfssl/cmd/cfssljson@latest" and add ~/go/bin to $PATH.' + if ! command -v cfssljson >/dev/null; then + die "Missing required program 'cfssljson'. ${howto_install}" + fi +} check_bats_core check_curl check_daemonizer +check_cfssl +check_cfssljson check_jq check_nc check_base64 check_python3 +check_yq check_pkill diff --git a/test/lib/setup_file.sh b/test/lib/setup_file.sh index a4231c98e..22a9bc303 100755 --- a/test/lib/setup_file.sh +++ b/test/lib/setup_file.sh @@ -127,24 +127,6 @@ is_db_sqlite() { } export -f is_db_sqlite -crowdsec_log() { - echo "$(config_get .common.log_dir)"/crowdsec.log -} -export -f crowdsec_log - -truncate_log() { - true > "$(crowdsec_log)" -} -export -f truncate_log - -assert_log() { - local oldout="${output:-}" - output="$(cat "$(crowdsec_log)")" - assert_output "$@" - output="${oldout}" -} -export -f assert_log - # Compare ignoring the key order, and allow "expected" without quoted identifiers. # Preserve the output variable in case the following commands require it. assert_json() { diff --git a/test/tools/.do-not-remove b/test/tools/.do-not-remove deleted file mode 100644 index 311928170..000000000 --- a/test/tools/.do-not-remove +++ /dev/null @@ -1 +0,0 @@ -this directory is populated by test dependencies, and is not checked in git