beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Revert "Merge remote-tracking branch 'origin' into coraza_poc_acquis"

Browse source 
This reverts commit 7098e971c7, reversing
changes made to 13512891e4.
This commit is contained in:
alteredCoder 2023-07-04 18:46:20 +02:00
parent 7098e971c7
commit 84b6570554
244 changed files with 2068 additions and 4722 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
.github/workflows/bats-hub.yml vendored
View file

@ -15,7 +15,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.20.4"]
name: "Build + tests"
runs-on: ubuntu-latest
@ -27,26 +27,40 @@ jobs:
sudo chmod +w /etc/machine-id
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: "Check out CrowdSec repository"
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: true
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: Cache Go modules
uses: actions/cache@v3
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: "Install bats dependencies"
env:
GOBIN: /usr/local/bin
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd
go install github.com/mikefarah/yq/v4@latest
go install github.com/cloudflare/cfssl/cmd/cfssl@master
go install github.com/cloudflare/cfssl/cmd/cfssljson@master
- name: "Build crowdsec and fixture"
run: make bats-clean bats-build bats-fixture BUILD_STATIC=1
run: make bats-clean bats-build bats-fixture
- name: "Run hub tests"
run: make bats-test-hub

28
.github/workflows/bats-mysql.yml vendored
View file

@ -14,7 +14,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.20.4"]
name: "Build + tests"
runs-on: ubuntu-latest
@ -34,27 +34,41 @@ jobs:
sudo chmod +w /etc/machine-id
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: "Check out CrowdSec repository"
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: true
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: Cache Go modules
uses: actions/cache@v3
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: "Install bats dependencies"
env:
GOBIN: /usr/local/bin
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd
go install github.com/mikefarah/yq/v4@latest
go install github.com/cloudflare/cfssl/cmd/cfssl@master
go install github.com/cloudflare/cfssl/cmd/cfssljson@master
- name: "Build crowdsec and fixture"
run: |
make clean bats-build bats-fixture BUILD_STATIC=1
make clean bats-build bats-fixture
env:
DB_BACKEND: mysql
MYSQL_HOST: 127.0.0.1

28
.github/workflows/bats-postgres.yml vendored
View file

@ -10,7 +10,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.20.4"]
name: "Build + tests"
runs-on: ubuntu-latest
@ -35,27 +35,41 @@ jobs:
sudo chmod +w /etc/machine-id
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: "Check out CrowdSec repository"
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: true
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: Cache Go modules
uses: actions/cache@v3
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: "Install bats dependencies"
env:
GOBIN: /usr/local/bin
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd
go install github.com/mikefarah/yq/v4@latest
go install github.com/cloudflare/cfssl/cmd/cfssl@master
go install github.com/cloudflare/cfssl/cmd/cfssljson@master
- name: "Build crowdsec and fixture (DB_BACKEND: pgx)"
run: |
make clean bats-build bats-fixture BUILD_STATIC=1
make clean bats-build bats-fixture
env:
DB_BACKEND: pgx
PGHOST: 127.0.0.1

28
.github/workflows/bats-sqlite-coverage.yml vendored
View file

@ -11,7 +11,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.20.4"]
name: "Build + tests"
runs-on: ubuntu-latest
@ -24,27 +24,41 @@ jobs:
sudo chmod +w /etc/machine-id
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: "Check out CrowdSec repository"
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: true
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: Cache Go modules
uses: actions/cache@v3
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: "Install bats dependencies"
env:
GOBIN: /usr/local/bin
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd
go install github.com/mikefarah/yq/v4@latest
go install github.com/cloudflare/cfssl/cmd/cfssl@master
go install github.com/cloudflare/cfssl/cmd/cfssljson@master
- name: "Build crowdsec and fixture"
run: |
make clean bats-build bats-fixture BUILD_STATIC=1
make clean bats-build bats-fixture
- name: "Run tests"
run: make bats-test

35
.github/workflows/cache-cleanup.yaml vendored
View file

@ -1,35 +0,0 @@
# https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#managing-caches
name: cleanup caches by a branch
on:
pull_request:
types:
- closed
jobs:
cleanup:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v3
- name: Cleanup
run: |
gh extension install actions/gh-actions-cache
REPO=${{ github.repository }}
BRANCH="refs/pull/${{ github.event.pull_request.number }}/merge"
echo "Fetching list of cache key"
cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH | cut -f 1 )
## Setting this to not fail the workflow while deleting cache keys.
set +e
echo "Deleting caches..."
for cacheKey in $cacheKeysForPR
do
gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm
done
echo "Done"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

23
.github/workflows/ci-windows-build-msi.yml vendored
View file

@ -23,27 +23,38 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.20.4"]
name: Build
runs-on: windows-2019
steps:
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: Check out code into the Go module directory
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: false
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: Cache Go modules
uses: actions/cache@v3
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: Build
run: make windows_installer BUILD_RE2_WASM=1
run: make windows_installer
- name: Upload MSI
uses: actions/upload-artifact@v3
with:

12
.github/workflows/docker-tests.yml vendored
View file

@ -30,6 +30,18 @@ jobs:
with:
config: .github/buildkit.toml
# - name: "Build flavor: full"
# uses: docker/build-push-action@v4
# with:
# context: .
# file: ./Dockerfile
# tags: crowdsecurity/crowdsec:test
# target: full
# platforms: linux/amd64
# load: true
# cache-from: type=gha
# cache-to: type=gha,mode=min
- name: "Build flavor: slim"
uses: docker/build-push-action@v4
with:

23
.github/workflows/go-tests-windows.yml vendored
View file

@ -22,28 +22,39 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.20.4"]
name: "Build + tests"
runs-on: windows-2022
steps:
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: Check out CrowdSec repository
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: false
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: Cache Go modules
uses: actions/cache@v3
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: Build
run: |
make build BUILD_RE2_WASM=1
make build
- name: Run tests
run: |

31
.github/workflows/go-tests.yml vendored
View file

@ -34,7 +34,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.20.4"]
name: "Build + tests"
runs-on: ubuntu-latest
@ -110,30 +110,35 @@ jobs:
steps:
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: Check out CrowdSec repository
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: false
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: Cache Go modules
uses: actions/cache@v3
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: Build and run tests, static
- name: Build and run tests
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev
go install github.com/ory/go-acc@v0.2.8
go install github.com/kyoh86/richgo@v0.3.10
set -o pipefail
make build BUILD_STATIC=1
make go-acc | richgo testfilter
- name: Run tests again, dynamic
run: |
make clean build
make build
make go-acc | richgo testfilter
- name: Upload unit coverage to Codecov

27
.github/workflows/release_publish-package.yml vendored
View file

@ -14,32 +14,41 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.20.4"]
name: Build and upload binary package
runs-on: ubuntu-latest
steps:
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: Check out code into the Go module directory
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: false
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: Cache Go modules
uses: actions/cache@v3
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: Build the binaries
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev
make vendor release BUILD_STATIC=1
run: make release
- name: Upload to release
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
tag_name="${GITHUB_REF##*/}"
hub release edit -a crowdsec-release.tgz -a vendor.tgz -m "" "$tag_name"
hub release edit -a crowdsec-release.tgz -m "" "$tag_name"

9
.gitignore vendored
View file

@ -15,9 +15,6 @@
*.test
*.cover
# Test dependencies
test/tools/*
# VMs used for dev/test
.vagrant
@ -33,10 +30,8 @@ test/coverage/*
*.swp
*.swo
# Dependencies are not vendored by default, but a tarball is created by "make vendor"
# and provided in the release. Used by freebsd, gentoo, etc.
vendor/
vendor.tgz
# Dependency directories (remove the comment below to include it)
# vendor/
# crowdsec binaries
cmd/crowdsec-cli/cscli

2
.golangci.yml
View file

@ -65,6 +65,7 @@ linters:
# - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers
# - bidichk # Checks for dangerous unicode character sequences
# - decorder # check declaration order and count of types, constants, variables and functions
# - depguard # Go linter that checks if package imports are in a list of acceptable packages
# - dupword # checks for duplicate words in the source code
# - durationcheck # check for two durations multiplied together
# - errcheck # Errcheck is a program for checking for unchecked errors in go programs. These unchecked errors can be critical bugs in some cases
@ -124,7 +125,6 @@ linters:
- thelper # thelper detects golang test helpers without t.Helper() call and checks the consistency of test helpers
- wastedassign # wastedassign finds wasted assignment statements.
- wrapcheck # Checks that errors returned from external packages are wrapped
- depguard # Go linter that checks if package imports are in a list of acceptable packages
#
# Recommended? (requires some work)

20
Dockerfile
View file

@ -1,31 +1,33 @@
# vim: set ft=dockerfile:
ARG GOVERSION=1.20.5
ARG GOVERSION=1.20.4
FROM golang:${GOVERSION}-alpine AS build
WORKDIR /go/src/crowdsec
# We like to choose the release of re2 to use, and Alpine does not ship a static version anyway.
COPY . .
# Alpine does not ship a static version of re2, we can build it ourselves
# Later versions require 'abseil', which is likewise not available in its static form
ENV RE2_VERSION=2023-03-01
# wizard.sh requires GNU coreutils
RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils pkgconfig && \
RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils icu-static re2-dev pkgconfig && \
wget https://github.com/google/re2/archive/refs/tags/${RE2_VERSION}.tar.gz && \
tar -xzf ${RE2_VERSION}.tar.gz && \
cd re2-${RE2_VERSION} && \
make && \
make install && \
echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
go install github.com/mikefarah/yq/v4@v4.34.1
COPY . .
RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
cd - && \
make clean release DOCKER_BUILD=1 && \
cd crowdsec-v* && \
./wizard.sh --docker-mode && \
cd - >/dev/null && \
cscli hub update && \
cscli collections install crowdsecurity/linux && \
cscli parsers install crowdsecurity/whitelists
cscli parsers install crowdsecurity/whitelists && \
go install github.com/mikefarah/yq/v4@v4.31.2
# In case we need to remove agents here..
# cscli machines list -o json | yq '.[].machineId' | xargs -r cscli machines delete

30
Dockerfile.debian
View file

@ -1,41 +1,32 @@
# vim: set ft=dockerfile:
ARG GOVERSION=1.20.5
ARG GOVERSION=1.20.4
FROM golang:${GOVERSION}-bookworm AS build
FROM golang:${GOVERSION}-bullseye AS build
WORKDIR /go/src/crowdsec
COPY . .
ENV DEBIAN_FRONTEND=noninteractive
ENV DEBCONF_NOWARNINGS="yes"
# We like to choose the release of re2 to use, the debian version is usually older.
ENV RE2_VERSION=2023-03-01
# wizard.sh requires GNU coreutils
RUN apt-get update && \
apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata && \
wget https://github.com/google/re2/archive/refs/tags/${RE2_VERSION}.tar.gz && \
tar -xzf ${RE2_VERSION}.tar.gz && \
cd re2-${RE2_VERSION} && \
make && \
make install && \
apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata libre2-dev && \
echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
go install github.com/mikefarah/yq/v4@v4.34.1
COPY . .
RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
make clean release DOCKER_BUILD=1 && \
cd crowdsec-v* && \
./wizard.sh --docker-mode && \
cd - >/dev/null && \
cscli hub update && \
cscli collections install crowdsecurity/linux && \
cscli parsers install crowdsecurity/whitelists
cscli parsers install crowdsecurity/whitelists && \
go install github.com/mikefarah/yq/v4@v4.31.2
# In case we need to remove agents here..
# cscli machines list -o json | yq '.[].machineId' | xargs -r cscli machines delete
FROM debian:bookworm-slim as slim
FROM debian:bullseye-slim as slim
ENV DEBIAN_FRONTEND=noninteractive
ENV DEBCONF_NOWARNINGS="yes"
@ -53,6 +44,9 @@ RUN apt-get update && \
mkdir -p /staging/var/lib/crowdsec && \
mkdir -p /var/lib/crowdsec/data
RUN echo "deb http://deb.debian.org/debian bullseye-backports main" >> /etc/apt/sources.list \
&& apt-get update && apt-get install -t bullseye-backports -y libsystemd0
COPY --from=build /go/bin/yq /usr/local/bin/yq
COPY --from=build /etc/crowdsec /staging/etc/crowdsec
COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec

99
Makefile
View file

@ -1,36 +1,8 @@
include mk/platform.mk
include mk/gmsl
# By default, this build requires the C++ re2 library to be installed.
#
# Debian/Ubuntu: apt install libre2-dev
# Fedora/CentOS: dnf install re2-devel
# FreeBSD: pkg install re2
# Alpine: apk add re2-dev
# Windows: choco install re2
# MacOS: brew install re2
# To build without re2, run "make BUILD_RE2_WASM=1"
# The WASM version is slower and introduces a short delay when starting a process
# (including cscli) so it is not recommended for production use.
BUILD_RE2_WASM ?= 0
# To build static binaries, run "make BUILD_STATIC=1".
# On some platforms, this requires additional packages
# (e.g. glibc-static and libstdc++-static on fedora, centos.. which are on the powertools/crb repository).
# If the static build fails at the link stage, it might be because the static library is not provided
# for your distribution (look for libre2.a). See the Dockerfile for an example of how to build it.
BUILD_STATIC ?= 0
# List of plugins to build
PLUGINS ?= $(patsubst ./plugins/notifications/%,%,$(wildcard ./plugins/notifications/*))
# Can be overriden, if you can deal with the consequences
BUILD_REQUIRE_GO_MAJOR ?= 1
BUILD_REQUIRE_GO_MINOR ?= 20
#--------------------------------------
GOCMD = go
GOTEST = $(GOCMD) test
@ -38,6 +10,8 @@ BUILD_CODENAME ?= alphaga
CROWDSEC_FOLDER = ./cmd/crowdsec
CSCLI_FOLDER = ./cmd/crowdsec-cli/
PLUGINS ?= $(patsubst ./plugins/notifications/%,%,$(wildcard ./plugins/notifications/*))
PLUGINS_DIR = ./plugins/notifications
CROWDSEC_BIN = crowdsec$(EXT)
@ -48,14 +22,8 @@ RELDIR = crowdsec-$(BUILD_VERSION)
GO_MODULE_NAME = github.com/crowdsecurity/crowdsec
# Check if a given value is considered truthy and returns "0" or "1".
# A truthy value is one of the following: "1", "yes", or "true", case-insensitive.
#
# Usage:
# ifeq ($(call bool,$(FOO)),1)
# $(info Let's foo)
# endif
bool = $(if $(filter $(call lc, $1),1 yes true),1,0)
# see if we have libre2-dev installed for C++ optimizations
RE2_CHECK := $(shell pkg-config --libs re2 2>/dev/null)
#--------------------------------------
#
@ -78,30 +46,13 @@ endif
GO_TAGS := netgo,osusergo,sqlite_omit_load_extension
ifeq ($(call bool,$(BUILD_RE2_WASM)),0)
ifeq ($(PKG_CONFIG),)
$(error "pkg-config is not available. Please install pkg-config.")
endif
ifeq ($(RE2_CHECK),)
# we could detect the platform and suggest the command to install
RE2_FAIL := "libre2-dev is not installed, please install it or set BUILD_RE2_WASM=1 to use the WebAssembly version"
else
ifneq (,$(RE2_CHECK))
# += adds a space that we don't want
GO_TAGS := $(GO_TAGS),re2_cgo
LD_OPTS_VARS += -X '$(GO_MODULE_NAME)/pkg/cwversion.Libre2=C++'
endif
endif
ifeq ($(call bool,$(BUILD_STATIC)),1)
BUILD_TYPE = static
EXTLDFLAGS := -extldflags '-static'
else
BUILD_TYPE = dynamic
EXTLDFLAGS :=
endif
export LD_OPTS=-ldflags "-s -w $(EXTLDFLAGS) $(LD_OPTS_VARS)" \
export LD_OPTS=-ldflags "-s -w -extldflags '-static' $(LD_OPTS_VARS)" \
-trimpath -tags $(GO_TAGS)
ifneq (,$(TEST_COVERAGE))
@ -113,15 +64,12 @@ endif
.PHONY: build
build: pre-build goversion crowdsec cscli plugins
# Sanity checks and build information
.PHONY: pre-build
pre-build:
$(info Building $(BUILD_VERSION) ($(BUILD_TAG)) $(BUILD_TYPE) for $(GOOS)/$(GOARCH))
ifneq (,$(RE2_FAIL))
$(error $(RE2_FAIL))
ifdef BUILD_STATIC
$(warning WARNING: The BUILD_STATIC variable is deprecated and has no effect. Builds are static by default since v1.5.0.)
endif
$(info Building $(BUILD_VERSION) ($(BUILD_TAG)) for $(GOOS)/$(GOARCH))
ifneq (,$(RE2_CHECK))
$(info Using C++ regexp library)
else
@ -165,7 +113,6 @@ testclean: bats-clean
@$(RM) pkg/cwhub/install $(WIN_IGNORE_ERR)
@$(RM) pkg/types/example.txt $(WIN_IGNORE_ERR)
# for the tests with localstack
export AWS_ENDPOINT_FORCE=http://localhost:4566
export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
@ -173,18 +120,15 @@ export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
testenv:
@echo 'NOTE: You need Docker, docker-compose and run "make localstack" in a separate shell ("make localstack-stop" to terminate it)'
# run the tests with localstack
.PHONY: test
test: testenv goversion
$(GOTEST) $(LD_OPTS) ./...
# run the tests with localstack and coverage
.PHONY: go-acc
go-acc: testenv goversion
go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models -- $(LD_OPTS) | \
sed 's/ *coverage:.*of statements in.*//'
# mock AWS services
.PHONY: localstack
localstack:
docker-compose -f test/localstack/docker-compose.yml up
@ -193,27 +137,13 @@ localstack:
localstack-stop:
docker-compose -f test/localstack/docker-compose.yml down
# list of plugins that contain go.mod
PLUGIN_VENDOR = $(foreach plugin,$(PLUGINS),$(shell if [ -f $(PLUGINS_DIR)/$(plugin)/go.mod ]; then echo $(PLUGINS_DIR)/$(plugin); fi))
# build vendor.tgz to be distributed with the release
.PHONY: vendor
vendor:
$(foreach plugin_dir,$(PLUGIN_VENDOR), \
cd $(plugin_dir) >/dev/null && \
$(GOCMD) mod vendor && \
cd - >/dev/null; \
@echo "Vendoring dependencies"
@$(GOCMD) mod vendor
@$(foreach plugin,$(PLUGINS), \
$(MAKE) -C $(PLUGINS_DIR)/$(plugin) vendor $(MAKE_FLAGS); \
)
$(GOCMD) mod vendor
tar -czf vendor.tgz vendor $(foreach plugin_dir,$(PLUGIN_VENDOR),$(plugin_dir)/vendor)
# remove vendor directories and vendor.tgz
.PHONY: vendor-remove
vendor-remove:
$(foreach plugin_dir,$(PLUGIN_VENDOR), \
$(RM) $(plugin_dir)/vendor; \
)
$(RM) vendor vendor.tgz
.PHONY: package
package:
@ -244,16 +174,13 @@ else
@if (Test-Path -Path $(RELDIR)) { echo "$(RELDIR) already exists, abort" ; exit 1 ; }
endif
# build a release tarball
.PHONY: release
release: check_release build package
# build the windows installer
.PHONY: windows_installer
windows_installer: build
@.\make_installer.ps1 -version $(BUILD_VERSION)
# build the chocolatey package
.PHONY: chocolatey
chocolatey: windows_installer
@.\make_chocolatey.ps1 -version $(BUILD_VERSION)

4
azure-pipelines.yml
View file

@ -27,7 +27,7 @@ stages:
- task: GoTool@0
displayName: "Install Go 1.20"
inputs:
version: '1.20.5'
version: '1.20.4'
- pwsh: |
choco install -y make
@ -38,7 +38,7 @@ stages:
pwsh: true
#we are not calling make windows_installer because we want to sign the binaries before they are added to the MSI
script: |
make build BUILD_RE2_WASM=1
make build
- task: AzureKeyVault@2
inputs:
azureSubscription: 'Azure subscription 1(8a93ab40-7e99-445e-ad47-0f6a3e2ef546)'

8
cmd/crowdsec-cli/alerts.go
View file

@ -15,6 +15,7 @@ import (
"github.com/fatih/color"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
@ -154,6 +155,7 @@ var alertTemplate = `
`
func DisplayOneAlert(alert *models.Alert, withDetail bool) error {
if csConfig.Cscli.Output == "human" {
tmpl, err := template.New("alert").Parse(alertTemplate)
@ -209,11 +211,11 @@ func NewAlertsCmd() *cobra.Command {
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
var err error
if err := csConfig.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %w", err)
return errors.Wrap(err, "loading api client")
}
apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL)
if err != nil {
return fmt.Errorf("parsing api url %s: %w", apiURL, err)
return errors.Wrapf(err, "parsing api url %s", apiURL)
}
Client, err = apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Client.Credentials.Login,
@ -224,7 +226,7 @@ func NewAlertsCmd() *cobra.Command {
})
if err != nil {
return fmt.Errorf("new api client: %w", err)
return errors.Wrap(err, "new api client")
}
return nil
},

19
cmd/crowdsec-cli/capi.go
View file

@ -6,11 +6,6 @@ import (
"net/url"
"os"
"github.com/go-openapi/strfmt"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
@ -19,6 +14,11 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/fflag"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
)
const CAPIBaseURL string = "https://api.crowdsec.net/"
@ -31,11 +31,8 @@ func NewCapiCmd() *cobra.Command {
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadAPIServer(); err != nil {
return fmt.Errorf("local API is disabled, please run this command on the local API machine: %w", err)
}
if csConfig.DisableAPI {
return nil
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
return errors.Wrap(err, "Local API is disabled, please run this command on the local API machine")
}
if csConfig.API.Server.OnlineClient == nil {
log.Fatalf("no configuration for Central API in '%s'", *csConfig.FilePath)
@ -136,7 +133,7 @@ func NewCapiStatusCmd() *cobra.Command {
Run: func(cmd *cobra.Command, args []string) {
var err error
if csConfig.API.Server == nil {
log.Fatal("There is no configuration on 'api.server:'")
log.Fatalln("There is no configuration on 'api.server:'")
}
if csConfig.API.Server.OnlineClient == nil {
log.Fatalf("Please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)

42
cmd/crowdsec-cli/config_backup.go
View file

@ -5,9 +5,11 @@ import (
"os"
"path/filepath"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
@ -32,17 +34,17 @@ func backupConfigToDirectory(dirPath string) error {
/*if parent directory doesn't exist, bail out. create final dir with Mkdir*/
parentDir := filepath.Dir(dirPath)
if _, err := os.Stat(parentDir); err != nil {
return fmt.Errorf("while checking parent directory %s existence: %w", parentDir, err)
return errors.Wrapf(err, "while checking parent directory %s existence", parentDir)
}
if err = os.Mkdir(dirPath, 0o700); err != nil {
return fmt.Errorf("while creating %s: %w", dirPath, err)
return errors.Wrapf(err, "while creating %s", dirPath)
}
if csConfig.ConfigPaths.SimulationFilePath != "" {
backupSimulation := filepath.Join(dirPath, "simulation.yaml")
if err = CopyFile(csConfig.ConfigPaths.SimulationFilePath, backupSimulation); err != nil {
return fmt.Errorf("failed copy %s to %s: %w", csConfig.ConfigPaths.SimulationFilePath, backupSimulation, err)
if err = types.CopyFile(csConfig.ConfigPaths.SimulationFilePath, backupSimulation); err != nil {
return errors.Wrapf(err, "failed copy %s to %s", csConfig.ConfigPaths.SimulationFilePath, backupSimulation)
}
log.Infof("Saved simulation to %s", backupSimulation)
@ -54,14 +56,14 @@ func backupConfigToDirectory(dirPath string) error {
*/
if csConfig.Crowdsec != nil && csConfig.Crowdsec.AcquisitionFilePath != "" {
backupAcquisition := filepath.Join(dirPath, "acquis.yaml")
if err = CopyFile(csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil {
return fmt.Errorf("failed copy %s to %s: %s", csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition, err)
if err = types.CopyFile(csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition, err)
}
}
acquisBackupDir := filepath.Join(dirPath, "acquis")
if err = os.Mkdir(acquisBackupDir, 0o700); err != nil {
return fmt.Errorf("error while creating %s: %s", acquisBackupDir, err)
return fmt.Errorf("error while creating %s : %s", acquisBackupDir, err)
}
if csConfig.Crowdsec != nil && len(csConfig.Crowdsec.AcquisitionFiles) > 0 {
@ -73,11 +75,11 @@ func backupConfigToDirectory(dirPath string) error {
targetFname, err := filepath.Abs(filepath.Join(acquisBackupDir, filepath.Base(acquisFile)))
if err != nil {
return fmt.Errorf("while saving %s to %s: %w", acquisFile, acquisBackupDir, err)
return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir)
}
if err = CopyFile(acquisFile, targetFname); err != nil {
return fmt.Errorf("failed copy %s to %s: %w", acquisFile, targetFname, err)
if err = types.CopyFile(acquisFile, targetFname); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err)
}
log.Infof("Saved acquis %s to %s", acquisFile, targetFname)
@ -86,8 +88,8 @@ func backupConfigToDirectory(dirPath string) error {
if ConfigFilePath != "" {
backupMain := fmt.Sprintf("%s/config.yaml", dirPath)
if err = CopyFile(ConfigFilePath, backupMain); err != nil {
return fmt.Errorf("failed copy %s to %s: %s", ConfigFilePath, backupMain, err)
if err = types.CopyFile(ConfigFilePath, backupMain); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", ConfigFilePath, backupMain, err)
}
log.Infof("Saved default yaml to %s", backupMain)
@ -95,8 +97,8 @@ func backupConfigToDirectory(dirPath string) error {
if csConfig.API != nil && csConfig.API.Server != nil && csConfig.API.Server.OnlineClient != nil && csConfig.API.Server.OnlineClient.CredentialsFilePath != "" {
backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath)
if err = CopyFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds); err != nil {
return fmt.Errorf("failed copy %s to %s: %s", csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds, err)
if err = types.CopyFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds, err)
}
log.Infof("Saved online API credentials to %s", backupCAPICreds)
@ -104,8 +106,8 @@ func backupConfigToDirectory(dirPath string) error {
if csConfig.API != nil && csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" {
backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath)
if err = CopyFile(csConfig.API.Client.CredentialsFilePath, backupLAPICreds); err != nil {
return fmt.Errorf("failed copy %s to %s: %s", csConfig.API.Client.CredentialsFilePath, backupLAPICreds, err)
if err = types.CopyFile(csConfig.API.Client.CredentialsFilePath, backupLAPICreds); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Client.CredentialsFilePath, backupLAPICreds, err)
}
log.Infof("Saved local API credentials to %s", backupLAPICreds)
@ -113,20 +115,21 @@ func backupConfigToDirectory(dirPath string) error {
if csConfig.API != nil && csConfig.API.Server != nil && csConfig.API.Server.ProfilesPath != "" {
backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath)
if err = CopyFile(csConfig.API.Server.ProfilesPath, backupProfiles); err != nil {
return fmt.Errorf("failed copy %s to %s: %s", csConfig.API.Server.ProfilesPath, backupProfiles, err)
if err = types.CopyFile(csConfig.API.Server.ProfilesPath, backupProfiles); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Server.ProfilesPath, backupProfiles, err)
}
log.Infof("Saved profiles to %s", backupProfiles)
}
if err = BackupHub(dirPath); err != nil {
return fmt.Errorf("failed to backup hub config: %s", err)
return fmt.Errorf("failed to backup hub config : %s", err)
}
return nil
}
func runConfigBackup(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadHub(); err != nil {
return err
@ -144,6 +147,7 @@ func runConfigBackup(cmd *cobra.Command, args []string) error {
return nil
}
func NewConfigBackupCmd() *cobra.Command {
cmdConfigBackup := &cobra.Command{
Use: `backup "directory"`,

24
cmd/crowdsec-cli/config_restore.go
View file

@ -7,11 +7,13 @@ import (
"os"
"path/filepath"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
@ -36,7 +38,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
backupMain := fmt.Sprintf("%s/config.yaml", dirPath)
if _, err = os.Stat(backupMain); err == nil {
if csConfig.ConfigPaths != nil && csConfig.ConfigPaths.ConfigDir != "" {
if err = CopyFile(backupMain, fmt.Sprintf("%s/config.yaml", csConfig.ConfigPaths.ConfigDir)); err != nil {
if err = types.CopyFile(backupMain, fmt.Sprintf("%s/config.yaml", csConfig.ConfigPaths.ConfigDir)); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", backupMain, csConfig.ConfigPaths.ConfigDir, err)
}
}
@ -49,21 +51,21 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath)
if _, err = os.Stat(backupCAPICreds); err == nil {
if err = CopyFile(backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath); err != nil {
if err = types.CopyFile(backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath, err)
}
}
backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath)
if _, err = os.Stat(backupLAPICreds); err == nil {
if err = CopyFile(backupLAPICreds, csConfig.API.Client.CredentialsFilePath); err != nil {
if err = types.CopyFile(backupLAPICreds, csConfig.API.Client.CredentialsFilePath); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", backupLAPICreds, csConfig.API.Client.CredentialsFilePath, err)
}
}
backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath)
if _, err = os.Stat(backupProfiles); err == nil {
if err = CopyFile(backupProfiles, csConfig.API.Server.ProfilesPath); err != nil {
if err = types.CopyFile(backupProfiles, csConfig.API.Server.ProfilesPath); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", backupProfiles, csConfig.API.Server.ProfilesPath, err)
}
}
@ -104,7 +106,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
backupSimulation := fmt.Sprintf("%s/simulation.yaml", dirPath)
if _, err = os.Stat(backupSimulation); err == nil {
if err = CopyFile(backupSimulation, csConfig.ConfigPaths.SimulationFilePath); err != nil {
if err = types.CopyFile(backupSimulation, csConfig.ConfigPaths.SimulationFilePath); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", backupSimulation, csConfig.ConfigPaths.SimulationFilePath, err)
}
}
@ -121,7 +123,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
if _, err = os.Stat(backupAcquisition); err == nil {
log.Debugf("restoring backup'ed %s", backupAcquisition)
if err = CopyFile(backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath); err != nil {
if err = types.CopyFile(backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath, err)
}
}
@ -132,12 +134,12 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
for _, acquisFile := range acquisFiles {
targetFname, err := filepath.Abs(csConfig.Crowdsec.AcquisitionDirPath + "/" + filepath.Base(acquisFile))
if err != nil {
return fmt.Errorf("while saving %s to %s: %w", acquisFile, targetFname, err)
return errors.Wrapf(err, "while saving %s to %s", acquisFile, targetFname)
}
log.Debugf("restoring %s to %s", acquisFile, targetFname)
if err = CopyFile(acquisFile, targetFname); err != nil {
if err = types.CopyFile(acquisFile, targetFname); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err)
}
}
@ -155,10 +157,10 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
targetFname, err := filepath.Abs(filepath.Join(acquisBackupDir, filepath.Base(acquisFile)))
if err != nil {
return fmt.Errorf("while saving %s to %s: %w", acquisFile, acquisBackupDir, err)
return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir)
}
if err = CopyFile(acquisFile, targetFname); err != nil {
if err = types.CopyFile(acquisFile, targetFname); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err)
}
@ -173,6 +175,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
return nil
}
func runConfigRestore(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
@ -197,6 +200,7 @@ func runConfigRestore(cmd *cobra.Command, args []string) error {
return nil
}
func NewConfigRestoreCmd() *cobra.Command {
cmdConfigRestore := &cobra.Command{
Use: `restore "directory"`,

73
cmd/crowdsec-cli/copyfile.go
View file

@ -1,73 +0,0 @@
package main
import (
"fmt"
"io"
"os"
"path/filepath"
log "github.com/sirupsen/logrus"
)
/*help to copy the file, ioutil doesn't offer the feature*/
func copyFileContents(src, dst string) (err error) {
in, err := os.Open(src)
if err != nil {
return
}
defer in.Close()
out, err := os.Create(dst)
if err != nil {
return
}
defer func() {
cerr := out.Close()
if err == nil {
err = cerr
}
}()
if _, err = io.Copy(out, in); err != nil {
return
}
err = out.Sync()
return
}
/*copy the file, ioutile doesn't offer the feature*/
func CopyFile(sourceSymLink, destinationFile string) (err error) {
sourceFile, err := filepath.EvalSymlinks(sourceSymLink)
if err != nil {
log.Infof("Not a symlink : %s", err)
sourceFile = sourceSymLink
}
sourceFileStat, err := os.Stat(sourceFile)
if err != nil {
return
}
if !sourceFileStat.Mode().IsRegular() {
// cannot copy non-regular files (e.g., directories,
// symlinks, devices, etc.)
return fmt.Errorf("copyFile: non-regular source file %s (%q)", sourceFileStat.Name(), sourceFileStat.Mode().String())
}
destinationFileStat, err := os.Stat(destinationFile)
if err != nil {
if !os.IsNotExist(err) {
return
}
} else {
if !(destinationFileStat.Mode().IsRegular()) {
return fmt.Errorf("copyFile: non-regular destination file %s (%q)", destinationFileStat.Name(), destinationFileStat.Mode().String())
}
if os.SameFile(sourceFileStat, destinationFileStat) {
return
}
}
if err = os.Link(sourceFile, destinationFile); err != nil {
err = copyFileContents(sourceFile, destinationFile)
}
return
}

245
cmd/crowdsec-cli/decisions.go
View file

@ -7,15 +7,19 @@ import (
"fmt"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/fatih/color"
"github.com/go-openapi/strfmt"
"github.com/jszwec/csvutil"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/go-cs-lib/pkg/ptr"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
@ -108,12 +112,12 @@ func NewDecisionsCmd() *cobra.Command {
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %w", err)
return errors.Wrap(err, "loading api client")
}
password := strfmt.Password(csConfig.API.Client.Credentials.Password)
apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL)
if err != nil {
return fmt.Errorf("parsing api url %s: %w", csConfig.API.Client.Credentials.URL, err)
return errors.Wrapf(err, "parsing api url %s", csConfig.API.Client.Credentials.URL)
}
Client, err = apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Client.Credentials.Login,
@ -123,7 +127,7 @@ func NewDecisionsCmd() *cobra.Command {
VersionPrefix: "v1",
})
if err != nil {
return fmt.Errorf("creating api client: %w", err)
return errors.Wrap(err, "creating api client")
}
return nil
},
@ -165,11 +169,11 @@ cscli decisions list -t ban
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
Run: func(cmd *cobra.Command, args []string) {
var err error
/*take care of shorthand options*/
if err = manageCliDecisionAlerts(filter.IPEquals, filter.RangeEquals, filter.ScopeEquals, filter.ValueEquals); err != nil {
return err
if err := manageCliDecisionAlerts(filter.IPEquals, filter.RangeEquals, filter.ScopeEquals, filter.ValueEquals); err != nil {
log.Fatalf("%s", err)
}
filter.ActiveDecisionEquals = new(bool)
*filter.ActiveDecisionEquals = true
@ -185,7 +189,7 @@ cscli decisions list -t ban
days, err := strconv.Atoi(realDuration)
if err != nil {
printHelp(cmd)
return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until)
log.Fatalf("Can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until)
}
*filter.Until = fmt.Sprintf("%d%s", days*24, "h")
}
@ -198,7 +202,7 @@ cscli decisions list -t ban
days, err := strconv.Atoi(realDuration)
if err != nil {
printHelp(cmd)
return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Since)
log.Fatalf("Can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until)
}
*filter.Since = fmt.Sprintf("%d%s", days*24, "h")
}
@ -234,15 +238,13 @@ cscli decisions list -t ban
alerts, _, err := Client.Alerts.List(context.Background(), filter)
if err != nil {
return fmt.Errorf("unable to retrieve decisions: %w", err)
log.Fatalf("Unable to list decisions : %v", err)
}
err = DecisionsToTable(alerts, printMachine)
if err != nil {
return fmt.Errorf("unable to print decisions: %w", err)
log.Fatalf("unable to list decisions : %v", err)
}
return nil
},
}
cmdDecisionsList.Flags().SortFlags = false
@ -286,7 +288,7 @@ cscli decisions add --scope username --value foobar
/*TBD : fix long and example*/
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
Run: func(cmd *cobra.Command, args []string) {
var err error
alerts := models.AddAlertsRequest{}
origin := types.CscliOrigin
@ -301,7 +303,7 @@ cscli decisions add --scope username --value foobar
/*take care of shorthand options*/
if err := manageCliDecisionAlerts(&addIP, &addRange, &addScope, &addValue); err != nil {
return err
log.Fatalf("%s", err)
}
if addIP != "" {
@ -312,7 +314,7 @@ cscli decisions add --scope username --value foobar
addScope = types.Range
} else if addValue == "" {
printHelp(cmd)
return fmt.Errorf("Missing arguments, a value is required (--ip, --range or --scope and --value)")
log.Fatalf("Missing arguments, a value is required (--ip, --range or --scope and --value)")
}
if addReason == "" {
@ -355,11 +357,10 @@ cscli decisions add --scope username --value foobar
_, _, err = Client.Alerts.Add(context.Background(), alerts)
if err != nil {
return err
log.Fatal(err)
}
log.Info("Decision successfully added")
return nil
},
}
@ -400,27 +401,25 @@ cscli decisions delete --id 42
cscli decisions delete --type captcha
`,
/*TBD : refaire le Long/Example*/
PreRunE: func(cmd *cobra.Command, args []string) error {
PreRun: func(cmd *cobra.Command, args []string) {
if delDecisionAll {
return nil
return
}
if *delFilter.ScopeEquals == "" && *delFilter.ValueEquals == "" &&
*delFilter.TypeEquals == "" && *delFilter.IPEquals == "" &&
*delFilter.RangeEquals == "" && *delFilter.ScenarioEquals == "" &&
*delFilter.OriginEquals == "" && delDecisionId == "" {
cmd.Usage()
return fmt.Errorf("at least one filter or --all must be specified")
log.Fatalln("At least one filter or --all must be specified")
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
Run: func(cmd *cobra.Command, args []string) {
var err error
var decisions *models.DeleteDecisionResponse
/*take care of shorthand options*/
if err = manageCliDecisionAlerts(delFilter.IPEquals, delFilter.RangeEquals, delFilter.ScopeEquals, delFilter.ValueEquals); err != nil {
return err
if err := manageCliDecisionAlerts(delFilter.IPEquals, delFilter.RangeEquals, delFilter.ScopeEquals, delFilter.ValueEquals); err != nil {
log.Fatalf("%s", err)
}
if *delFilter.ScopeEquals == "" {
delFilter.ScopeEquals = nil
@ -450,19 +449,18 @@ cscli decisions delete --type captcha
if delDecisionId == "" {
decisions, _, err = Client.Decisions.Delete(context.Background(), delFilter)
if err != nil {
return fmt.Errorf("Unable to delete decisions: %v", err)
log.Fatalf("Unable to delete decisions : %v", err)
}
} else {
if _, err = strconv.Atoi(delDecisionId); err != nil {
return fmt.Errorf("id '%s' is not an integer: %v", delDecisionId, err)
log.Fatalf("id '%s' is not an integer: %v", delDecisionId, err)
}
decisions, _, err = Client.Decisions.DeleteOne(context.Background(), delDecisionId)
if err != nil {
return fmt.Errorf("Unable to delete decision: %v", err)
log.Fatalf("Unable to delete decision : %v", err)
}
}
log.Infof("%s decision(s) deleted", decisions.NbDeleted)
return nil
},
}
@ -480,3 +478,192 @@ cscli decisions delete --type captcha
return cmdDecisionsDelete
}
func NewDecisionsImportCmd() *cobra.Command {
var (
defaultDuration = "4h"
defaultScope = "ip"
defaultType = "ban"
defaultReason = "manual"
importDuration string
importScope string
importReason string
importType string
importFile string
batchSize int
)
var cmdDecisionImport = &cobra.Command{
Use: "import [options]",
Short: "Import decisions from json or csv file",
Long: "expected format :\n" +
"csv : any of duration,origin,reason,scope,type,value, with a header line\n" +
`json : {"duration" : "24h", "origin" : "my-list", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"}`,
DisableAutoGenTag: true,
Example: `decisions.csv :
duration,scope,value
24h,ip,1.2.3.4
cscsli decisions import -i decisions.csv
decisions.json :
[{"duration" : "4h", "scope" : "ip", "type" : "ban", "value" : "1.2.3.4"}]
`,
Run: func(cmd *cobra.Command, args []string) {
if importFile == "" {
log.Fatalf("Please provide a input file containing decisions with -i flag")
}
csvData, err := os.ReadFile(importFile)
if err != nil {
log.Fatalf("unable to open '%s': %s", importFile, err)
}
type decisionRaw struct {
Duration string `csv:"duration,omitempty" json:"duration,omitempty"`
Origin string `csv:"origin,omitempty" json:"origin,omitempty"`
Scenario string `csv:"reason,omitempty" json:"reason,omitempty"`
Scope string `csv:"scope,omitempty" json:"scope,omitempty"`
Type string `csv:"type,omitempty" json:"type,omitempty"`
Value string `csv:"value" json:"value"`
}
var decisionsListRaw []decisionRaw
switch fileFormat := filepath.Ext(importFile); fileFormat {
case ".json":
if err := json.Unmarshal(csvData, &decisionsListRaw); err != nil {
log.Fatalf("unable to unmarshall json: '%s'", err)
}
case ".csv":
if err := csvutil.Unmarshal(csvData, &decisionsListRaw); err != nil {
log.Fatalf("unable to unmarshall csv: '%s'", err)
}
default:
log.Fatalf("file format not supported for '%s'. supported format are 'json' and 'csv'", importFile)
}
decisionsList := make([]*models.Decision, 0)
for i, decisionLine := range decisionsListRaw {
line := i + 2
if decisionLine.Value == "" {
log.Fatalf("please provide a 'value' in your csv line %d", line)
}
/*deal with defaults and cli-override*/
if decisionLine.Duration == "" {
decisionLine.Duration = defaultDuration
log.Debugf("No 'duration' line %d, using default value: '%s'", line, defaultDuration)
}
if importDuration != "" {
decisionLine.Duration = importDuration
log.Debugf("'duration' line %d, using supplied value: '%s'", line, importDuration)
}
decisionLine.Origin = types.CscliImportOrigin
if decisionLine.Scenario == "" {
decisionLine.Scenario = defaultReason
log.Debugf("No 'reason' line %d, using value: '%s'", line, decisionLine.Scenario)
}
if importReason != "" {
decisionLine.Scenario = importReason
log.Debugf("No 'reason' line %d, using supplied value: '%s'", line, importReason)
}
if decisionLine.Type == "" {
decisionLine.Type = defaultType
log.Debugf("No 'type' line %d, using default value: '%s'", line, decisionLine.Type)
}
if importType != "" {
decisionLine.Type = importType
log.Debugf("'type' line %d, using supplied value: '%s'", line, importType)
}
if decisionLine.Scope == "" {
decisionLine.Scope = defaultScope
log.Debugf("No 'scope' line %d, using default value: '%s'", line, decisionLine.Scope)
}
if importScope != "" {
decisionLine.Scope = importScope
log.Debugf("'scope' line %d, using supplied value: '%s'", line, importScope)
}
decision := models.Decision{
Value: ptr.Of(decisionLine.Value),
Duration: ptr.Of(decisionLine.Duration),
Origin: ptr.Of(decisionLine.Origin),
Scenario: ptr.Of(decisionLine.Scenario),
Type: ptr.Of(decisionLine.Type),
Scope: ptr.Of(decisionLine.Scope),
Simulated: new(bool),
}
decisionsList = append(decisionsList, &decision)
}
alerts := models.AddAlertsRequest{}
if batchSize > 0 {
for i := 0; i < len(decisionsList); i += batchSize {
end := i + batchSize
if end > len(decisionsList) {
end = len(decisionsList)
}
decisionBatch := decisionsList[i:end]
importAlert := models.Alert{
CreatedAt: time.Now().UTC().Format(time.RFC3339),
Scenario: ptr.Of(fmt.Sprintf("import %s : %d IPs", importFile, len(decisionBatch))),
Message: ptr.Of(""),
Events: []*models.Event{},
Source: &models.Source{
Scope: ptr.Of(""),
Value: ptr.Of(""),
},
StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
Capacity: ptr.Of(int32(0)),
Simulated: ptr.Of(false),
EventsCount: ptr.Of(int32(len(decisionBatch))),
Leakspeed: ptr.Of(""),
ScenarioHash: ptr.Of(""),
ScenarioVersion: ptr.Of(""),
Decisions: decisionBatch,
}
alerts = append(alerts, &importAlert)
}
} else {
importAlert := models.Alert{
CreatedAt: time.Now().UTC().Format(time.RFC3339),
Scenario: ptr.Of(fmt.Sprintf("import %s : %d IPs", importFile, len(decisionsList))),
Message: ptr.Of(""),
Events: []*models.Event{},
Source: &models.Source{
Scope: ptr.Of(""),
Value: ptr.Of(""),
},
StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
Capacity: ptr.Of(int32(0)),
Simulated: ptr.Of(false),
EventsCount: ptr.Of(int32(len(decisionsList))),
Leakspeed: ptr.Of(""),
ScenarioHash: ptr.Of(""),
ScenarioVersion: ptr.Of(""),
Decisions: decisionsList,
}
alerts = append(alerts, &importAlert)
}
if len(decisionsList) > 1000 {
log.Infof("You are about to add %d decisions, this may take a while", len(decisionsList))
}
_, _, err = Client.Alerts.Add(context.Background(), alerts)
if err != nil {
log.Fatal(err)
}
log.Infof("%d decisions successfully imported", len(decisionsList))
},
}
cmdDecisionImport.Flags().SortFlags = false
cmdDecisionImport.Flags().StringVarP(&importFile, "input", "i", "", "Input file")
cmdDecisionImport.Flags().StringVarP(&importDuration, "duration", "d", "", "Decision duration (ie. 1h,4h,30m)")
cmdDecisionImport.Flags().StringVar(&importScope, "scope", types.Ip, "Decision scope (ie. ip,range,username)")
cmdDecisionImport.Flags().StringVarP(&importReason, "reason", "R", "", "Decision reason (ie. scenario-name)")
cmdDecisionImport.Flags().StringVarP(&importType, "type", "t", "", "Decision type (ie. ban,captcha,throttle)")
cmdDecisionImport.Flags().IntVar(&batchSize, "batch", 0, "Split import in batches of N decisions")
return cmdDecisionImport
}

272
cmd/crowdsec-cli/decisions_import.go
View file

@ -1,272 +0,0 @@
package main
import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"os"
"strings"
"time"
"github.com/jszwec/csvutil"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/go-cs-lib/pkg/ptr"
"github.com/crowdsecurity/go-cs-lib/pkg/slicetools"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
// decisionRaw is only used to unmarshall json/csv decisions
type decisionRaw struct {
Duration string `csv:"duration,omitempty" json:"duration,omitempty"`
Scenario string `csv:"reason,omitempty" json:"reason,omitempty"`
Scope string `csv:"scope,omitempty" json:"scope,omitempty"`
Type string `csv:"type,omitempty" json:"type,omitempty"`
Value string `csv:"value" json:"value"`
}
func parseDecisionList(content []byte, format string) ([]decisionRaw, error) {
ret := []decisionRaw{}
switch format {
case "values":
log.Infof("Parsing values")
scanner := bufio.NewScanner(bytes.NewReader(content))
for scanner.Scan() {
value := strings.TrimSpace(scanner.Text())
ret = append(ret, decisionRaw{Value: value})
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("unable to parse values: '%s'", err)
}
case "json":
log.Infof("Parsing json")
if err := json.Unmarshal(content, &ret); err != nil {
return nil, err
}
case "csv":
log.Infof("Parsing csv")
if err := csvutil.Unmarshal(content, &ret); err != nil {
return nil, fmt.Errorf("unable to parse csv: '%s'", err)
}
default:
return nil, fmt.Errorf("invalid format '%s', expected one of 'json', 'csv', 'values'", format)
}
return ret, nil
}
func runDecisionsImport(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
input, err := flags.GetString("input")
if err != nil {
return err
}
defaultDuration, err := flags.GetString("duration")
if err != nil {
return err
}
if defaultDuration == "" {
return fmt.Errorf("--duration cannot be empty")
}
defaultScope, err := flags.GetString("scope")
if err != nil {
return err
}
if defaultScope == "" {
return fmt.Errorf("--scope cannot be empty")
}
defaultReason, err := flags.GetString("reason")
if err != nil {
return err
}
if defaultReason == "" {
return fmt.Errorf("--reason cannot be empty")
}
defaultType, err := flags.GetString("type")
if err != nil {
return err
}
if defaultType == "" {
return fmt.Errorf("--type cannot be empty")
}
batchSize, err := flags.GetInt("batch")
if err != nil {
return err
}
format, err := flags.GetString("format")
if err != nil {
return err
}
var (
content []byte
fin *os.File
)
// set format if the file has a json or csv extension
if format == "" {
if strings.HasSuffix(input, ".json") {
format = "json"
} else if strings.HasSuffix(input, ".csv") {
format = "csv"
}
}
if format == "" {
return fmt.Errorf("unable to guess format from file extension, please provide a format with --format flag")
}
if input == "-" {
fin = os.Stdin
input = "stdin"
} else {
fin, err = os.Open(input)
if err != nil {
return fmt.Errorf("unable to open %s: %s", input, err)
}
}
content, err = io.ReadAll(fin)
if err != nil {
return fmt.Errorf("unable to read from %s: %s", input, err)
}
decisionsListRaw, err := parseDecisionList(content, format)
if err != nil {
return err
}
decisions := make([]*models.Decision, len(decisionsListRaw))
for i, d := range decisionsListRaw {
if d.Value == "" {
return fmt.Errorf("item %d: missing 'value'", i)
}
if d.Duration == "" {
d.Duration = defaultDuration
log.Debugf("item %d: missing 'duration', using default '%s'", i, defaultDuration)
}
if d.Scenario == "" {
d.Scenario = defaultReason
log.Debugf("item %d: missing 'reason', using default '%s'", i, defaultReason)
}
if d.Type == "" {
d.Type = defaultType
log.Debugf("item %d: missing 'type', using default '%s'", i, defaultType)
}
if d.Scope == "" {
d.Scope = defaultScope
log.Debugf("item %d: missing 'scope', using default '%s'", i, defaultScope)
}
decisions[i] = &models.Decision{
Value: ptr.Of(d.Value),
Duration: ptr.Of(d.Duration),
Origin: ptr.Of(types.CscliImportOrigin),
Scenario: ptr.Of(d.Scenario),
Type: ptr.Of(d.Type),
Scope: ptr.Of(d.Scope),
Simulated: ptr.Of(false),
}
}
alerts := models.AddAlertsRequest{}
for _, chunk := range slicetools.Chunks(decisions, batchSize) {
log.Debugf("Processing chunk of %d decisions", len(chunk))
importAlert := models.Alert{
CreatedAt: time.Now().UTC().Format(time.RFC3339),
Scenario: ptr.Of(fmt.Sprintf("import %s: %d IPs", input, len(chunk))),
Message: ptr.Of(""),
Events: []*models.Event{},
Source: &models.Source{
Scope: ptr.Of(""),
Value: ptr.Of(""),
},
StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
Capacity: ptr.Of(int32(0)),
Simulated: ptr.Of(false),
EventsCount: ptr.Of(int32(len(chunk))),
Leakspeed: ptr.Of(""),
ScenarioHash: ptr.Of(""),
ScenarioVersion: ptr.Of(""),
Decisions: chunk,
}
alerts = append(alerts, &importAlert)
}
if len(decisions) > 1000 {
log.Infof("You are about to add %d decisions, this may take a while", len(decisions))
}
_, _, err = Client.Alerts.Add(context.Background(), alerts)
if err != nil {
return err
}
log.Infof("Imported %d decisions", len(decisions))
return nil
}
func NewDecisionsImportCmd() *cobra.Command {
var cmdDecisionsImport = &cobra.Command{
Use: "import [options]",
Short: "Import decisions from a file or pipe",
Long: "expected format:\n" +
"csv : any of duration,reason,scope,type,value, with a header line\n" +
`json : {"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"}`,
DisableAutoGenTag: true,
Example: `decisions.csv:
duration,scope,value
24h,ip,1.2.3.4
$ cscli decisions import -i decisions.csv
decisions.json:
[{"duration" : "4h", "scope" : "ip", "type" : "ban", "value" : "1.2.3.4"}]
The file format is detected from the extension, but can be forced with the --format option
which is required when reading from standard input.
Raw values, standard input:
$ echo "1.2.3.4" | cscli decisions import -i - --format values
`,
RunE: runDecisionsImport,
}
flags := cmdDecisionsImport.Flags()
flags.SortFlags = false
flags.StringP("input", "i", "", "Input file")
flags.StringP("duration", "d", "4h", "Decision duration: 1h,4h,30m")
flags.String("scope", types.Ip, "Decision scope: ip,range,username")
flags.StringP("reason", "R", "manual", "Decision reason: <scenario-name>")
flags.StringP("type", "t", "ban", "Decision type: ban,captcha,throttle")
flags.Int("batch", 0, "Split import in batches of N decisions")
flags.String("format", "", "Input format: 'json', 'csv' or 'values' (each line is a value, no headers)")
cmdDecisionsImport.MarkFlagRequired("input")
return cmdDecisionsImport
}

6
cmd/crowdsec-cli/lapi.go
View file

@ -9,6 +9,7 @@ import (
"strings"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"golang.org/x/exp/slices"
@ -23,6 +24,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
var LAPIURLPrefix string = "v1"
@ -203,7 +205,7 @@ func NewLapiCmd() *cobra.Command {
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %w", err)
return errors.Wrap(err, "loading api client")
}
return nil
},
@ -438,7 +440,7 @@ cscli lapi context delete --value evt.Line.Src
return cmdContext
}
func detectStaticField(GrokStatics []parser.ExtraField) []string {
func detectStaticField(GrokStatics []types.ExtraField) []string {
ret := make([]string, 0)
for _, static := range GrokStatics {
if static.Parsed != "" {

33
cmd/crowdsec-cli/machines.go
View file

@ -12,6 +12,7 @@ import (
"time"
"github.com/AlecAivazis/survey/v2"
"github.com/enescakir/emoji"
"github.com/fatih/color"
"github.com/go-openapi/strfmt"
"github.com/google/uuid"
@ -84,21 +85,22 @@ func generateID(prefix string) (string, error) {
return prefix + suffix, nil
}
// getLastHeartbeat returns the last heartbeat timestamp of a machine
// and a boolean indicating if the machine is considered active or not.
func getLastHeartbeat(m *ent.Machine) (string, bool) {
if m.LastHeartbeat == nil {
return "-", false
func displayLastHeartBeat(m *ent.Machine, fancy bool) string {
var hbDisplay string
if m.LastHeartbeat != nil {
lastHeartBeat := time.Now().UTC().Sub(*m.LastHeartbeat)
hbDisplay = lastHeartBeat.Truncate(time.Second).String()
if fancy && lastHeartBeat > 2*time.Minute {
hbDisplay = fmt.Sprintf("%s %s", emoji.Warning.String(), lastHeartBeat.Truncate(time.Second).String())
}
elapsed := time.Now().UTC().Sub(*m.LastHeartbeat)
hb := elapsed.Truncate(time.Second).String()
if elapsed > 2*time.Minute {
return hb, false
} else {
hbDisplay = "-"
if fancy {
hbDisplay = emoji.Warning.String() + " -"
}
return hb, true
}
return hbDisplay
}
func getAgents(out io.Writer, dbClient *database.Client) error {
@ -128,10 +130,9 @@ func getAgents(out io.Writer, dbClient *database.Client) error {
} else {
validated = "false"
}
hb, _ := getLastHeartbeat(m)
err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb})
err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, displayLastHeartBeat(m, false)})
if err != nil {
return fmt.Errorf("failed to write raw output: %w", err)
return fmt.Errorf("failed to write raw output : %s", err)
}
}
csvwriter.Flush()

6
cmd/crowdsec-cli/machines_table.go
View file

@ -24,11 +24,7 @@ func getAgentsTable(out io.Writer, machines []*ent.Machine) {
validated = emoji.Prohibited.String()
}
hb, active := getLastHeartbeat(m)
if !active {
hb = emoji.Warning.String() + " " + hb
}
t.AddRow(m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb)
t.AddRow(m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, displayLastHeartBeat(m, true))
}
t.Render()

58
cmd/crowdsec-cli/notifications.go
View file

@ -15,6 +15,7 @@ import (
"github.com/fatih/color"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/tomb.v2"
@ -27,12 +28,14 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/csprofiles"
)
type NotificationsCfg struct {
Config csplugin.PluginConfig `json:"plugin_config"`
Profiles []*csconfig.ProfileCfg `json:"associated_profiles"`
ids []uint
}
func NewNotificationsCmd() *cobra.Command {
var cmdNotifications = &cobra.Command{
Use: "notifications [action]",
@ -54,6 +57,7 @@ func NewNotificationsCmd() *cobra.Command {
},
}
cmdNotifications.AddCommand(NewNotificationsListCmd())
cmdNotifications.AddCommand(NewNotificationsInspectCmd())
cmdNotifications.AddCommand(NewNotificationsReinjectCmd())
@ -61,17 +65,18 @@ func NewNotificationsCmd() *cobra.Command {
return cmdNotifications
}
func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
pcfgs := map[string]csplugin.PluginConfig{}
wf := func(path string, info fs.FileInfo, err error) error {
if info == nil {
return fmt.Errorf("error while traversing directory %s: %w", path, err)
return errors.Wrapf(err, "error while traversing directory %s", path)
}
name := filepath.Join(csConfig.ConfigPaths.NotificationDir, info.Name()) //Avoid calling info.Name() twice
if (strings.HasSuffix(name, "yaml") || strings.HasSuffix(name, "yml")) && !(info.IsDir()) {
ts, err := csplugin.ParsePluginConfigFile(name)
if err != nil {
return fmt.Errorf("loading notifification plugin configuration with %s: %w", name, err)
return errors.Wrapf(err, "Loading notifification plugin configuration with %s", name)
}
for _, t := range ts {
pcfgs[t.Name] = t
@ -81,14 +86,14 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
}
if err := filepath.Walk(csConfig.ConfigPaths.NotificationDir, wf); err != nil {
return nil, fmt.Errorf("while loading notifification plugin configuration: %w", err)
return nil, errors.Wrap(err, "Loading notifification plugin configuration")
}
// A bit of a tricky stuf now: reconcile profiles and notification plugins
ncfgs := map[string]NotificationsCfg{}
profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles)
if err != nil {
return nil, fmt.Errorf("while extracting profiles from configuration: %w", err)
return nil, errors.Wrap(err, "Cannot extract profiles from configuration")
}
for profileID, profile := range profiles {
loop:
@ -124,6 +129,7 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
return ncfgs, nil
}
func NewNotificationsListCmd() *cobra.Command {
var cmdNotificationsList = &cobra.Command{
Use: "list",
@ -135,7 +141,7 @@ func NewNotificationsListCmd() *cobra.Command {
RunE: func(cmd *cobra.Command, arg []string) error {
ncfgs, err := getNotificationsConfiguration()
if err != nil {
return fmt.Errorf("can't build profiles configuration: %w", err)
return errors.Wrap(err, "Can't build profiles configuration")
}
if csConfig.Cscli.Output == "human" {
@ -143,14 +149,14 @@ func NewNotificationsListCmd() *cobra.Command {
} else if csConfig.Cscli.Output == "json" {
x, err := json.MarshalIndent(ncfgs, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal notification configuration: %w", err)
return errors.New("failed to marshal notification configuration")
}
fmt.Printf("%s", string(x))
} else if csConfig.Cscli.Output == "raw" {
csvwriter := csv.NewWriter(os.Stdout)
err := csvwriter.Write([]string{"Name", "Type", "Profile name"})
if err != nil {
return fmt.Errorf("failed to write raw header: %w", err)
return errors.Wrap(err, "failed to write raw header")
}
for _, b := range ncfgs {
profilesList := []string{}
@ -159,7 +165,7 @@ func NewNotificationsListCmd() *cobra.Command {
}
err := csvwriter.Write([]string{b.Config.Name, b.Config.Type, strings.Join(profilesList, ", ")})
if err != nil {
return fmt.Errorf("failed to write raw content: %w", err)
return errors.Wrap(err, "failed to write raw content")
}
}
csvwriter.Flush()
@ -171,6 +177,7 @@ func NewNotificationsListCmd() *cobra.Command {
return cmdNotificationsList
}
func NewNotificationsInspectCmd() *cobra.Command {
var cmdNotificationsInspect = &cobra.Command{
Use: "inspect",
@ -188,14 +195,14 @@ func NewNotificationsInspectCmd() *cobra.Command {
pluginName := arg[0]
if pluginName == "" {
return fmt.Errorf("please provide a plugin name to inspect")
errors.New("Please provide a plugin name to inspect")
}
ncfgs, err := getNotificationsConfiguration()
if err != nil {
return fmt.Errorf("can't build profiles configuration: %w", err)
return errors.Wrap(err, "Can't build profiles configuration")
}
if cfg, ok = ncfgs[pluginName]; !ok {
return fmt.Errorf("plugin '%s' does not exist or is not active", pluginName)
return errors.New("The provided plugin name doesn't exist or isn't active")
}
if csConfig.Cscli.Output == "human" || csConfig.Cscli.Output == "raw" {
@ -209,7 +216,7 @@ func NewNotificationsInspectCmd() *cobra.Command {
} else if csConfig.Cscli.Output == "json" {
x, err := json.MarshalIndent(cfg, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal notification configuration: %w", err)
return errors.New("failed to marshal notification configuration")
}
fmt.Printf("%s", string(x))
}
@ -220,6 +227,7 @@ func NewNotificationsInspectCmd() *cobra.Command {
return cmdNotificationsInspect
}
func NewNotificationsReinjectCmd() *cobra.Command {
var remediation bool
var alertOverride string
@ -242,26 +250,26 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
)
if len(args) != 1 {
printHelp(cmd)
return fmt.Errorf("wrong number of argument: there should be one argument")
return errors.New("Wrong number of argument: there should be one argument")
}
//first: get the alert
id, err := strconv.Atoi(args[0])
if err != nil {
return fmt.Errorf("bad alert id %s", args[0])
return errors.New(fmt.Sprintf("bad alert id %s", args[0]))
}
if err := csConfig.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %w", err)
return errors.Wrapf(err, "loading api client")
}
if csConfig.API.Client == nil {
return fmt.Errorf("missing configuration on 'api_client:'")
return errors.New("There is no configuration on 'api_client:'")
}
if csConfig.API.Client.Credentials == nil {
return fmt.Errorf("missing API credentials in '%s'", csConfig.API.Client.CredentialsFilePath)
return errors.New(fmt.Sprintf("Please provide credentials for the API in '%s'", csConfig.API.Client.CredentialsFilePath))
}
apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL)
if err != nil {
return fmt.Errorf("error parsing the URL of the API: %w", err)
return errors.Wrapf(err, "error parsing the URL of the API")
}
client, err := apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Client.Credentials.Login,
@ -271,16 +279,16 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
VersionPrefix: "v1",
})
if err != nil {
return fmt.Errorf("error creating the client for the API: %w", err)
return errors.Wrapf(err, "error creating the client for the API")
}
alert, _, err := client.Alerts.GetByID(context.Background(), id)
if err != nil {
return fmt.Errorf("can't find alert with id %s: %w", args[0], err)
return errors.Wrapf(err, fmt.Sprintf("can't find alert with id %s", args[0]))
}
if alertOverride != "" {
if err = json.Unmarshal([]byte(alertOverride), alert); err != nil {
return fmt.Errorf("can't unmarshal data in the alert flag: %w", err)
return errors.Wrapf(err, "Can't unmarshal the data given in the alert flag")
}
}
if !remediation {
@ -290,7 +298,7 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
// second we start plugins
err = pluginBroker.Init(csConfig.PluginConfig, csConfig.API.Server.Profiles, csConfig.ConfigPaths)
if err != nil {
return fmt.Errorf("can't initialize plugins: %w", err)
return errors.Wrapf(err, "Can't initialize plugins")
}
pluginTomb.Go(func() error {
@ -302,13 +310,13 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles)
if err != nil {
return fmt.Errorf("cannot extract profiles from configuration: %w", err)
return errors.Wrap(err, "Cannot extract profiles from configuration")
}
for id, profile := range profiles {
_, matched, err := profile.EvaluateProfile(alert)
if err != nil {
return fmt.Errorf("can't evaluate profile %s: %w", profile.Cfg.Name, err)
return errors.Wrapf(err, "can't evaluate profile %s", profile.Cfg.Name)
}
if !matched {
log.Infof("The profile %s didn't match", profile.Cfg.Name)
@ -336,7 +344,7 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
}
// time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
pluginTomb.Kill(fmt.Errorf("terminating"))
pluginTomb.Kill(errors.New("terminating"))
pluginTomb.Wait()
return nil
},

3
cmd/crowdsec-cli/scenarios.go
View file

@ -4,6 +4,7 @@ import (
"fmt"
"github.com/fatih/color"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
@ -32,7 +33,7 @@ cscli scenarios remove crowdsecurity/ssh-bf
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("while setting hub branch: %w", err)
return errors.Wrap(err, "while setting hub branch")
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {

11
cmd/crowdsec-cli/support.go
View file

@ -26,6 +26,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
const (
@ -47,14 +48,6 @@ const (
SUPPORT_CROWDSEC_PROFILE_PATH = "config/profiles.yaml"
)
// from https://github.com/acarl005/stripansi
var reStripAnsi = regexp.MustCompile("[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))")
func stripAnsiString(str string) string {
// the byte version doesn't strip correctly
return reStripAnsi.ReplaceAllString(str, "")
}
func collectMetrics() ([]byte, []byte, error) {
log.Info("Collecting prometheus metrics")
err := csConfig.LoadPrometheus()
@ -407,7 +400,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
log.Errorf("Could not add zip entry for %s: %s", filename, err)
continue
}
fw.Write([]byte(stripAnsiString(string(data))))
fw.Write([]byte(types.StripAnsiString(string(data))))
}
err = zipWriter.Close()

6
cmd/crowdsec-cli/utils.go
View file

@ -598,7 +598,7 @@ func RestoreHub(dirPath string) error {
log.Infof("Going to restore local/tainted [%s]", tfile.Name())
sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
if err = CopyFile(sourceFile, destinationFile); err != nil {
if err = types.CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
@ -607,7 +607,7 @@ func RestoreHub(dirPath string) error {
log.Infof("Going to restore local/tainted [%s]", file.Name())
sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
if err = CopyFile(sourceFile, destinationFile); err != nil {
if err = types.CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
@ -657,7 +657,7 @@ func BackupHub(dirPath string) error {
}
clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
if err = CopyFile(v.LocalPath, tfile); err != nil {
if err = types.CopyFile(v.LocalPath, tfile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
}
clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)

11
cmd/crowdsec/api.go
View file

@ -1,7 +1,6 @@
package main
import (
"fmt"
"runtime"
"time"
@ -21,7 +20,7 @@ func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) {
apiServer, err := apiserver.NewServer(cConfig.API.Server)
if err != nil {
return nil, fmt.Errorf("unable to run local API: %w", err)
return nil, errors.Wrap(err, "unable to run local API")
}
if hasPlugins(cConfig.API.Server.Profiles) {
@ -30,27 +29,23 @@ func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) {
if cConfig.PluginConfig == nil && runtime.GOOS != "windows" {
return nil, errors.New("plugins are enabled, but the plugin_config section is missing in the configuration")
}
if cConfig.ConfigPaths.NotificationDir == "" {
return nil, errors.New("plugins are enabled, but config_paths.notification_dir is not defined")
}
if cConfig.ConfigPaths.PluginDir == "" {
return nil, errors.New("plugins are enabled, but config_paths.plugin_dir is not defined")
}
err = pluginBroker.Init(cConfig.PluginConfig, cConfig.API.Server.Profiles, cConfig.ConfigPaths)
if err != nil {
return nil, fmt.Errorf("unable to run plugin broker: %w", err)
return nil, errors.Wrap(err, "unable to run local API")
}
log.Info("initiated plugin broker")
apiServer.AttachPluginBroker(&pluginBroker)
}
err = apiServer.InitController()
if err != nil {
return nil, fmt.Errorf("unable to run local API: %w", err)
return nil, errors.Wrap(err, "unable to run local API")
}
return apiServer, nil

19
cmd/crowdsec/crowdsec.go
View file

@ -3,12 +3,10 @@ package main
import (
"fmt"
"os"
"path/filepath"
"sync"
"time"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
"path/filepath"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
@ -18,28 +16,31 @@ import (
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
)
func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
var err error
// Populate cwhub package tools
if err = cwhub.GetHubIdx(cConfig.Hub); err != nil {
return nil, fmt.Errorf("while loading hub index: %w", err)
if err := cwhub.GetHubIdx(cConfig.Hub); err != nil {
return &parser.Parsers{}, fmt.Errorf("Failed to load hub index : %s", err)
}
// Start loading configs
csParsers := parser.NewParsers()
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
return nil, fmt.Errorf("while loading parsers: %w", err)
return &parser.Parsers{}, fmt.Errorf("Failed to load parsers: %s", err)
}
if err := LoadBuckets(cConfig); err != nil {
return nil, fmt.Errorf("while loading scenarios: %w", err)
return &parser.Parsers{}, fmt.Errorf("Failed to load scenarios: %s", err)
}
if err := LoadAcquisition(cConfig); err != nil {
return nil, fmt.Errorf("while loading acquisition config: %w", err)
return &parser.Parsers{}, fmt.Errorf("Error while loading acquisition config : %s", err)
}
return csParsers, nil
}
@ -117,7 +118,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
aggregated = true
}
if err := acquisition.GetMetrics(dataSources, aggregated); err != nil {
return fmt.Errorf("while fetching prometheus metrics for datasources: %w", err)
return errors.Wrap(err, "while fetching prometheus metrics for datasources.")
}
}

44
cmd/crowdsec/main.go
View file

@ -52,14 +52,11 @@ var (
type Flags struct {
ConfigFile string
LogLevelTrace bool
LogLevelDebug bool
LogLevelInfo bool
LogLevelWarn bool
LogLevelError bool
LogLevelFatal bool
TraceLevel bool
DebugLevel bool
InfoLevel bool
WarnLevel bool
ErrorLevel bool
PrintVersion bool
SingleFileType string
Labels map[string]string
@ -110,7 +107,7 @@ func LoadAcquisition(cConfig *csconfig.Config) error {
dataSources, err = acquisition.LoadAcquisitionFromDSN(flags.OneShotDSN, flags.Labels, flags.Transform)
if err != nil {
return fmt.Errorf("failed to configure datasource for %s: %w", flags.OneShotDSN, err)
return errors.Wrapf(err, "failed to configure datasource for %s", flags.OneShotDSN)
}
} else {
dataSources, err = acquisition.LoadAcquisitionFromFile(cConfig.Crowdsec)
@ -119,10 +116,6 @@ func LoadAcquisition(cConfig *csconfig.Config) error {
}
}
if len(dataSources) == 0 {
return fmt.Errorf("no datasource enabled")
}
return nil
}
@ -147,14 +140,11 @@ func (l labelsMap) Set(label string) error {
func (f *Flags) Parse() {
flag.StringVar(&f.ConfigFile, "c", csconfig.DefaultConfigPath("config.yaml"), "configuration file")
flag.BoolVar(&f.LogLevelTrace, "trace", false, "set log level to 'trace' (VERY verbose)")
flag.BoolVar(&f.LogLevelDebug, "debug", false, "set log level to 'debug'")
flag.BoolVar(&f.LogLevelInfo, "info", false, "set log level to 'info'")
flag.BoolVar(&f.LogLevelWarn, "warning", false, "set log level to 'warning'")
flag.BoolVar(&f.LogLevelError, "error", false, "set log level to 'error'")
flag.BoolVar(&f.LogLevelFatal, "fatal", false, "set log level to 'fatal'")
flag.BoolVar(&f.TraceLevel, "trace", false, "VERY verbose")
flag.BoolVar(&f.DebugLevel, "debug", false, "print debug-level on stderr")
flag.BoolVar(&f.InfoLevel, "info", false, "print info-level on stderr")
flag.BoolVar(&f.WarnLevel, "warning", false, "print warning-level on stderr")
flag.BoolVar(&f.ErrorLevel, "error", false, "print error-level on stderr")
flag.BoolVar(&f.PrintVersion, "version", false, "display version")
flag.StringVar(&f.OneShotDSN, "dsn", "", "Process a single data source in time-machine")
flag.StringVar(&f.Transform, "transform", "", "expr to apply on the event after acquisition")
@ -182,18 +172,16 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level {
// override from flags
switch {
case f.LogLevelTrace:
case f.TraceLevel:
ret = log.TraceLevel
case f.LogLevelDebug:
case f.DebugLevel:
ret = log.DebugLevel
case f.LogLevelInfo:
case f.InfoLevel:
ret = log.InfoLevel
case f.LogLevelWarn:
case f.WarnLevel:
ret = log.WarnLevel
case f.LogLevelError:
case f.ErrorLevel:
ret = log.ErrorLevel
case f.LogLevelFatal:
ret = log.FatalLevel
default:
}

15
cmd/crowdsec/output.go
View file

@ -7,10 +7,6 @@ import (
"sync"
"time"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
@ -20,6 +16,9 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
func dedupAlerts(alerts []types.RuntimeAlert) ([]*models.Alert, error) {
@ -51,11 +50,11 @@ func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error
alertsToPush, err := dedupAlerts(alerts)
if err != nil {
return fmt.Errorf("failed to transform alerts for api: %w", err)
return errors.Wrap(err, "failed to transform alerts for api")
}
_, _, err = client.Alerts.Add(ctx, alertsToPush)
if err != nil {
return fmt.Errorf("failed sending alert to LAPI: %w", err)
return errors.Wrap(err, "failed sending alert to LAPI")
}
return nil
}
@ -105,11 +104,11 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
Scenarios: scenarios,
})
if err != nil {
return fmt.Errorf("authenticate watcher (%s): %w", apiConfig.Login, err)
return errors.Wrapf(err, "authenticate watcher (%s)", apiConfig.Login)
}
if err := Client.GetClient().Transport.(*apiclient.JWTTransport).Expiration.UnmarshalText([]byte(authResp.Expire)); err != nil {
return fmt.Errorf("unable to parse jwt expiration: %w", err)
return errors.Wrap(err, "unable to parse jwt expiration")
}
Client.GetClient().Transport.(*apiclient.JWTTransport).Token = authResp.Token

4
cmd/crowdsec/pour.go
View file

@ -12,7 +12,9 @@ import (
)
func runPour(input chan types.Event, holders []leaky.BucketFactory, buckets *leaky.Buckets, cConfig *csconfig.Config) error {
count := 0
var (
count int
)
for {
//bucket is now ready
select {

13
cmd/crowdsec/run_in_svc_windows.go
View file

@ -3,6 +3,7 @@ package main
import (
"fmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"golang.org/x/sys/windows/svc"
@ -21,7 +22,7 @@ func StartRunSvc() error {
isRunninginService, err := svc.IsWindowsService()
if err != nil {
return fmt.Errorf("failed to determine if we are running in windows service mode: %w", err)
return errors.Wrap(err, "failed to determine if we are running in windows service mode")
}
if isRunninginService {
return runService(svcName)
@ -30,22 +31,22 @@ func StartRunSvc() error {
if flags.WinSvc == "Install" {
err = installService(svcName, svcDescription)
if err != nil {
return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err)
return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName)
}
} else if flags.WinSvc == "Remove" {
err = removeService(svcName)
if err != nil {
return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err)
return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName)
}
} else if flags.WinSvc == "Start" {
err = startService(svcName)
if err != nil {
return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err)
return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName)
}
} else if flags.WinSvc == "Stop" {
err = controlService(svcName, svc.Stop, svc.Stopped)
if err != nil {
return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err)
return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName)
}
} else if flags.WinSvc == "" {
return WindowsRun()
@ -65,7 +66,7 @@ func WindowsRun() error {
if err != nil {
return err
}
// Configure logging
log.Infof("Crowdsec %s", version.String())
apiReady := make(chan bool, 1)

35
cmd/crowdsec/serve.go
View file

@ -1,16 +1,16 @@
package main
import (
"fmt"
"os"
"os/signal"
"syscall"
"time"
"github.com/coreos/go-systemd/v22/daemon"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/csdaemon"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
@ -68,7 +68,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
}
apiServer, err := initAPIServer(cConfig)
if err != nil {
return nil, fmt.Errorf("unable to init api server: %w", err)
return nil, errors.Wrap(err, "unable to init api server")
}
apiReady := make(chan bool, 1)
@ -78,7 +78,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
if !cConfig.DisableAgent {
csParsers, err := initCrowdsec(cConfig)
if err != nil {
return nil, fmt.Errorf("unable to init crowdsec: %w", err)
return nil, errors.Wrap(err, "unable to init crowdsec")
}
// restore bucket state
@ -180,13 +180,13 @@ func shutdownCrowdsec() error {
func shutdown(sig os.Signal, cConfig *csconfig.Config) error {
if !cConfig.DisableAgent {
if err := shutdownCrowdsec(); err != nil {
return fmt.Errorf("failed to shut down crowdsec: %w", err)
return errors.Wrap(err, "failed to shut down crowdsec")
}
}
if !cConfig.DisableAPI {
if err := shutdownAPI(); err != nil {
return fmt.Errorf("failed to shut down api routines: %w", err)
return errors.Wrap(err, "failed to shut down api routines")
}
}
@ -238,13 +238,13 @@ func HandleSignals(cConfig *csconfig.Config) error {
log.Warning("SIGHUP received, reloading")
if err = shutdown(s, cConfig); err != nil {
exitChan <- fmt.Errorf("failed shutdown: %w", err)
exitChan <- errors.Wrap(err, "failed shutdown")
break Loop
}
if newConfig, err = reloadHandler(s); err != nil {
exitChan <- fmt.Errorf("reload handler failure: %w", err)
exitChan <- errors.Wrap(err, "reload handler failure")
break Loop
}
@ -256,7 +256,7 @@ func HandleSignals(cConfig *csconfig.Config) error {
case os.Interrupt, syscall.SIGTERM:
log.Warning("SIGTERM received, shutting down")
if err = shutdown(s, cConfig); err != nil {
exitChan <- fmt.Errorf("failed shutdown: %w", err)
exitChan <- errors.Wrap(err, "failed shutdown")
break Loop
}
@ -284,17 +284,17 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
if cConfig.API.Server != nil && cConfig.API.Server.DbConfig != nil {
dbClient, err := database.NewClient(cConfig.API.Server.DbConfig)
if err != nil {
return fmt.Errorf("failed to get database client: %w", err)
return errors.Wrap(err, "failed to get database client")
}
err = exprhelpers.Init(dbClient)
if err != nil {
return fmt.Errorf("failed to init expr helpers: %w", err)
return errors.Wrap(err, "failed to init expr helpers")
}
} else {
err := exprhelpers.Init(nil)
if err != nil {
return fmt.Errorf("failed to init expr helpers: %w", err)
return errors.Wrap(err, "failed to init expr helpers")
}
log.Warningln("Exprhelpers loaded without database client.")
@ -303,7 +303,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
if cConfig.API.CTI != nil && *cConfig.API.CTI.Enabled {
log.Infof("Crowdsec CTI helper enabled")
if err := exprhelpers.InitCrowdsecCTI(cConfig.API.CTI.Key, cConfig.API.CTI.CacheTimeout, cConfig.API.CTI.CacheSize, cConfig.API.CTI.LogLevel); err != nil {
return fmt.Errorf("failed to init crowdsec cti: %w", err)
return errors.Wrap(err, "failed to init crowdsec cti")
}
}
@ -319,7 +319,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
apiServer, err := initAPIServer(cConfig)
if err != nil {
return fmt.Errorf("api server init: %w", err)
return errors.Wrap(err, "api server init")
}
if !flags.TestMode {
@ -332,7 +332,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
if !cConfig.DisableAgent {
csParsers, err := initCrowdsec(cConfig)
if err != nil {
return fmt.Errorf("crowdsec init: %w", err)
return errors.Wrap(err, "crowdsec init")
}
// if it's just linting, we're done
@ -350,7 +350,10 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
}
if cConfig.Common != nil && cConfig.Common.Daemonize {
csdaemon.NotifySystemd(log.StandardLogger())
sent, err := daemon.SdNotify(false, daemon.SdNotifyReady)
if !sent || err != nil {
log.Errorf("Failed to notify(sent: %v): %v", sent, err)
}
// wait for signals
return HandleSignals(cConfig)
}

4
cmd/crowdsec/win_service.go
View file

@ -8,10 +8,10 @@
package main
import (
"fmt"
"syscall"
"time"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"golang.org/x/sys/windows"
"golang.org/x/sys/windows/svc"
@ -106,7 +106,7 @@ func runService(name string) error {
winsvc := crowdsec_winservice{config: cConfig}
if err := svc.Run(name, &winsvc); err != nil {
return fmt.Errorf("%s service failed: %w", name, err)
return errors.Wrapf(err, "%s service failed", name)
}
log.Infof("%s service stopped", name)

4
debian/control vendored
View file

@ -1,8 +1,6 @@
Source: crowdsec
Maintainer: Crowdsec Team <debian@crowdsec.net>
Build-Depends: debhelper, bash
Section: admin
Priority: optional
Build-Depends: debhelper, bash, git
Package: crowdsec
Architecture: any

12
debian/rules vendored
View file

@ -1,6 +1,6 @@
#!/usr/bin/make -f
export DEB_VERSION=$(shell dpkg-parsechangelog | grep -E '^Version:' | cut -f 2 -d ' ')
export DEB_VERSION=$(shell dpkg-parsechangelog | egrep '^Version:' | cut -f 2 -d ' ')
export BUILD_VERSION=v${DEB_VERSION}-debian-pragmatic
export GO111MODULE=on
@ -11,10 +11,12 @@ override_dh_auto_clean:
override_dh_auto_test:
override_dh_auto_build:
override_dh_auto_install:
# just use the prebuilt binaries, otherwise:
# make build BUILD_RE_WASM=0 BUILD_STATIC=1
# mkdir /tmp/go
# echo $(go version)
# echo $($GOCMD version)
# cd cmd/crowdsec && GOROOT=/tmp/go GO111MODULE=on $(GOBUILD) $(LD_OPTS) -o $(CROWDSEC_BIN) -v && cd ..
# cd cmd/crowdsec-cli && GOROOT=/tmp/go GO111MODULE=on $(GOBUILD) $(LD_OPTS) -o cscli -v && cd ..
make build
mkdir -p debian/crowdsec/usr/bin
mkdir -p debian/crowdsec/etc/crowdsec
mkdir -p debian/crowdsec/usr/share/crowdsec

2
debian/templates vendored
View file

@ -17,7 +17,7 @@ Description: Address of the local API server
Template: crowdsec/capi
Type: boolean
Default: true
Description: Do you want to use the centralized remote API server ?
Description: Do you want to the centralized remote API server ?
To share information with other crowdsec you can register to the centralized remote API server.
.
If you don't know what to do, answer yes.

2
docker/docker_start.sh
View file

@ -56,7 +56,7 @@ conf_get() {
if [ $# -ge 2 ]; then
yq e "$1" "$2"
else
cscli config show-yaml | yq e "$1"
yq e "$1" "$CONFIG_FILE"
fi
}

32
docker/test/tests/test_hub_collections.py
View file

@ -6,8 +6,11 @@ Test collection management
from http import HTTPStatus
import json
import os
import pwd
import pytest
import yaml
pytestmark = pytest.mark.docker
@ -82,7 +85,12 @@ def test_taint_bubble_up(crowdsec, tmp_path_factory, flavor):
'COLLECTIONS': f'{coll}'
}
with crowdsec(flavor=flavor, environment=env) as cs:
hub = tmp_path_factory.mktemp("hub")
volumes = {
hub: {'bind': '/etc/crowdsec/hub', 'mode': 'rw'}
}
with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs:
cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
res = cs.cont.exec_run('cscli collections list -o json')
assert res.exit_code == 0
@ -94,13 +102,25 @@ def test_taint_bubble_up(crowdsec, tmp_path_factory, flavor):
f'*Enabled collections : {coll}*',
])
scenario = 'crowdsecurity/http-crawl-non_statics'
# the description won't be read back, it's from the index
yq_command = f"yq -e -i '.description=\"tainted\"' /etc/crowdsec/hub/scenarios/{scenario}.yaml"
res = cs.cont.exec_run(yq_command)
# change file permissions to allow edit
current_uid = pwd.getpwuid(os.getuid()).pw_uid
res = cs.cont.exec_run(f'chown -R {current_uid} /etc/crowdsec/hub')
assert res.exit_code == 0
scenario = 'crowdsecurity/http-crawl-non_statics'
scenario_file = hub / f'scenarios/{scenario}.yaml'
with open(scenario_file) as f:
yml = yaml.safe_load(f)
yml['description'] += ' (tainted)'
# won't be able to read it back because description is taken from the index
with open(scenario_file, 'w') as f:
yaml.dump(yml, f)
with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs:
cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
res = cs.cont.exec_run(f'cscli scenarios inspect {scenario} -o json')
assert res.exit_code == 0
j = json.loads(res.output)

89
go.mod
View file

@ -5,22 +5,14 @@ go 1.20
require (
entgo.io/ent v0.11.3
github.com/AlecAivazis/survey/v2 v2.2.7
github.com/Masterminds/semver/v3 v3.1.1
github.com/Masterminds/sprig/v3 v3.2.2
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/alexliesenfeld/health v0.5.1
github.com/antonmedv/expr v1.12.5
github.com/appleboy/gin-jwt/v2 v2.8.0
github.com/aquasecurity/table v1.8.0
github.com/aws/aws-lambda-go v1.38.0
github.com/aws/aws-sdk-go v1.42.25
github.com/beevik/etree v1.1.0
github.com/blackfireio/osinfo v1.0.3
github.com/bluele/gcache v0.0.2
github.com/buger/jsonparser v1.1.1
github.com/c-robinson/iplib v1.0.3
github.com/cespare/xxhash/v2 v2.1.2
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26
github.com/crowdsecurity/go-cs-lib v0.0.2
github.com/crowdsecurity/grokky v0.2.1
github.com/crowdsecurity/machineid v1.0.2
github.com/davecgh/go-spew v1.1.1
@ -28,7 +20,7 @@ require (
github.com/docker/docker v20.10.24+incompatible
github.com/docker/go-connections v0.4.0
github.com/enescakir/emoji v1.0.0
github.com/fatih/color v1.15.0
github.com/fatih/color v1.13.0
github.com/fsnotify/fsnotify v1.6.0
github.com/gin-gonic/gin v1.7.7
github.com/go-co-op/gocron v1.17.0
@ -37,50 +29,65 @@ require (
github.com/go-openapi/swag v0.19.14
github.com/go-openapi/validate v0.20.0
github.com/go-sql-driver/mysql v1.6.0
github.com/goccy/go-yaml v1.9.7
github.com/gofrs/uuid v4.0.0+incompatible
github.com/golang-jwt/jwt/v4 v4.2.0
github.com/google/go-querystring v1.0.0
github.com/google/uuid v1.3.0
github.com/google/winops v0.0.0-20211216095627-f0e86eb1453b
github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e
github.com/hashicorp/go-hclog v1.0.0
github.com/hashicorp/go-plugin v1.4.2
github.com/hashicorp/go-version v1.2.1
github.com/ivanpirog/coloredcobra v1.0.1
github.com/jackc/pgx/v4 v4.14.1
github.com/jarcoal/httpmock v1.1.0
github.com/jszwec/csvutil v1.5.1
github.com/lithammer/dedent v1.1.0
github.com/mattn/go-isatty v0.0.17
github.com/mattn/go-sqlite3 v1.14.16
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826
github.com/nxadm/tail v1.4.8
github.com/oschwald/geoip2-golang v1.4.0
github.com/oschwald/maxminddb-golang v1.8.0
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.14.0
github.com/prometheus/client_model v0.3.0
github.com/prometheus/prom2json v1.3.0
github.com/r3labs/diff/v2 v2.14.1
github.com/segmentio/kafka-go v0.4.34
github.com/shirou/gopsutil/v3 v3.23.5
github.com/sirupsen/logrus v1.9.2
github.com/spf13/cobra v1.7.0
github.com/stretchr/testify v1.8.3
github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c
github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
github.com/wasilibs/go-re2 v0.2.1
golang.org/x/crypto v0.1.0
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
golang.org/x/mod v0.11.0
golang.org/x/sys v0.9.0
golang.org/x/mod v0.8.0
google.golang.org/grpc v1.47.0
google.golang.org/protobuf v1.28.1
gopkg.in/natefinch/lumberjack.v2 v2.2.1
gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637
gopkg.in/yaml.v2 v2.4.0
gotest.tools/v3 v3.0.3
)
require (
github.com/Masterminds/semver v1.5.0
github.com/Masterminds/sprig/v3 v3.2.2
github.com/aquasecurity/table v1.8.0
github.com/aws/aws-lambda-go v1.38.0
github.com/beevik/etree v1.1.0
github.com/blackfireio/osinfo v1.0.3
github.com/bluele/gcache v0.0.2
github.com/cespare/xxhash/v2 v2.1.2
github.com/corazawaf/coraza/v3 v3.0.0-00010101000000-000000000000
github.com/coreos/go-systemd/v22 v22.5.0
github.com/crowdsecurity/go-cs-lib v0.0.0-20230531105801-4c1535c2b3bd
github.com/goccy/go-yaml v1.9.7
github.com/gofrs/uuid v4.0.0+incompatible
github.com/golang-jwt/jwt/v4 v4.2.0
github.com/google/winops v0.0.0-20211216095627-f0e86eb1453b
github.com/ivanpirog/coloredcobra v1.0.1
github.com/lithammer/dedent v1.1.0
github.com/mattn/go-isatty v0.0.14
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
github.com/segmentio/kafka-go v0.4.34
github.com/shirou/gopsutil/v3 v3.22.12
github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c
github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
github.com/wasilibs/go-re2 v0.2.1
golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc
golang.org/x/sys v0.7.0
gopkg.in/yaml.v3 v3.0.1
k8s.io/apiserver v0.22.5
)
@ -88,7 +95,7 @@ require (
require (
ariga.io/atlas v0.7.2-0.20220927111110-867ee0cca56a // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/Masterminds/semver/v3 v3.1.1 // indirect
github.com/PuerkitoBio/purell v1.1.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/agext/levenshtein v1.2.1 // indirect
@ -96,7 +103,7 @@ require (
github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/corazawaf/libinjection-go v0.1.2 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/docker/distribution v2.8.2+incompatible // indirect
github.com/docker/go-units v0.4.0 // indirect
@ -140,7 +147,7 @@ require (
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/magefile/mage v1.14.0 // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
@ -155,6 +162,7 @@ require (
github.com/oklog/run v1.0.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 // indirect
github.com/pierrec/lz4/v4 v4.1.15 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
@ -163,24 +171,24 @@ require (
github.com/rivo/uniseg v0.2.0 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/shopspring/decimal v1.2.0 // indirect
github.com/spf13/cast v1.3.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/tetratelabs/wazero v1.0.0-rc.2 // indirect
github.com/tidwall/gjson v1.13.0 // indirect
github.com/tidwall/gjson v1.14.4 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
github.com/tklauser/go-sysconf v0.3.11 // indirect
github.com/tklauser/numcpus v0.6.0 // indirect
github.com/ugorji/go/codec v1.2.6 // indirect
github.com/vmihailenco/msgpack v4.0 .4+incompatible // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
github.com/zclconf/go-cty v1.8.0 // indirect
go.mongodb.org/mongo-driver v1.9.4 // indirect
golang.org/x/net v0.7.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/term v0.5.0 // indirect
golang.org/x/text v0.7.0 // indirect
golang.org/x/tools v0.6.0 // indirect
go.mongodb.org/mongo-driver v1.9.0 // indirect
golang.org/x/net v0.9.0 // indirect
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
golang.org/x/term v0.7.0 // indirect
golang.org/x/text v0.9.0 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
@ -190,8 +198,11 @@ require (
k8s.io/apimachinery v0.25.2 // indirect
k8s.io/klog/v2 v2.70.1 // indirect
k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect
rsc.io/binaryregexp v0.2.0 // indirect
sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
)
replace golang.org/x/time/rate => github.com/crowdsecurity/crowdsec/pkg/time/rate v0.0.0
replace github.com/corazawaf/coraza/v3 => ./coraza

85
go.sum
View file

@ -55,12 +55,14 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8=
github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk=
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8 h1:xzYJEypr/85nBpB11F9br+3HUrpgb+fcm5iADzXXYEw=
@ -149,6 +151,8 @@ github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMe
github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
github.com/corazawaf/libinjection-go v0.1.2 h1:oeiV9pc5rvJ+2oqOqXEAMJousPpGiup6f7Y3nZj5GoM=
github.com/corazawaf/libinjection-go v0.1.2/go.mod h1:OP4TM7xdJ2skyXqNX1AN1wN5nNZEmJNuWbNPOItn7aw=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
@ -170,8 +174,8 @@ github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
github.com/crowdsecurity/go-cs-lib v0.0.2 h1:+Tjmf/IclOXNzU9sxKVQvUl9CkMfbM60xQ0zA05NWps=
github.com/crowdsecurity/go-cs-lib v0.0.2/go.mod h1:iznTJ19qLTYdZBcRb5RVDlcUdSlayBCivBkWsXlOY3g=
github.com/crowdsecurity/go-cs-lib v0.0.0-20230531105801-4c1535c2b3bd h1:Y70ceDKAKYFXTnxEjXuBDSh07umvDhbX3PCCYhdtsZ0=
github.com/crowdsecurity/go-cs-lib v0.0.0-20230531105801-4c1535c2b3bd/go.mod h1:9JJLSpGj1ZXnROV3xAcJvS/HTaUvuA8K3gGOpO4tfVc=
github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4=
github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc=
@ -209,12 +213,12 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
@ -657,18 +661,16 @@ github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
@ -681,6 +683,7 @@ github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyex
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI=
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
@ -755,6 +758,8 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8=
github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
@ -831,12 +836,8 @@ github.com/segmentio/kafka-go v0.4.34 h1:Dm6YlLMiVSiwwav20KY0AoY63s661FXevwJ3CVH
github.com/segmentio/kafka-go v0.4.34/go.mod h1:GAjxBQJdQMB5zfNA21AhpaqOB2Mu+w3De4ni3Gbm8y0=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shirou/gopsutil/v3 v3.23.5 h1:5SgDCeQ0KW0S4N0znjeM/eFHXXOKyv2dVNgRq/c9P6Y=
github.com/shirou/gopsutil/v3 v3.23.5/go.mod h1:Ng3Maa27Q2KARVJ0SPZF5NdrQSC3XHKP8IIWrHgMeLY=
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
github.com/shirou/gopsutil/v3 v3.22.12 h1:oG0ns6poeUSxf78JtOsfygNWuEHYYz8hnnNg7P04TJs=
github.com/shirou/gopsutil/v3 v3.22.12/go.mod h1:Xd7P1kwZcp5VW52+9XsirIKd/BROzbb2wdX3Kqlz9uI=
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
@ -887,6 +888,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
@ -895,13 +897,14 @@ github.com/tetratelabs/wazero v1.0.0-rc.2/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+Gk
github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c h1:HelZ2kAFadG0La9d+4htN4HzQ68Bm2iM9qKMSMES6xg=
github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c/go.mod h1:JlzghshsemAMDGZLytTFY8C1JQxQPhnatWqNwUXjggo=
github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/gjson v1.13.0 h1:3TFY9yxOQShrvmjdM76K+jc66zJeT6D3/VFFYCGQf7M=
github.com/tidwall/gjson v1.13.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM=
github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms=
@ -940,8 +943,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zclconf/go-cty v1.8.0 h1:s4AvqaeQzJIu3ndv4gVIhplVD0krU+bgrcLSVUnaWuA=
github.com/zclconf/go-cty v1.8.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk=
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
@ -960,8 +963,8 @@ go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS
go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE=
go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
go.mongodb.org/mongo-driver v1.9.4 h1:qXWlnK2WCOWSxJ/Hm3XyYOGKv3ujA2btBsCyuIFvQjc=
go.mongodb.org/mongo-driver v1.9.4/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY=
go.mongodb.org/mongo-driver v1.9.0 h1:f3aLGJvQmBl8d9S40IL+jEyBC6hfLPbJjv9t5hEM9ck=
go.mongodb.org/mongo-driver v1.9.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@ -1028,8 +1031,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU=
golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -1052,8 +1055,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -1104,8 +1107,8 @@ golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -1125,8 +1128,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -1198,23 +1201,23 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -1224,8 +1227,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@ -1294,7 +1297,6 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -1465,6 +1467,7 @@ k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2R
k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4=
k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

969
mk/__gmsl
View file

@ -1,969 +0,0 @@
# ----------------------------------------------------------------------------
#
# GNU Make Standard Library (GMSL)
#
# A library of functions to be used with GNU Make's $(call) that
# provides functionality not available in standard GNU Make.
#
# Copyright (c) 2005-2022 John Graham-Cumming
#
# This file is part of GMSL
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# Neither the name of the John Graham-Cumming nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# ----------------------------------------------------------------------------
# This is the GNU Make Standard Library version number as a list with
# three items: major, minor, revision
gmsl_version := 1 2 0
__gmsl_name := GNU Make Standard Library
# Used to output warnings and error from the library, it's possible to
# disable any warnings or errors by overriding these definitions
# manually or by setting GMSL_NO_WARNINGS or GMSL_NO_ERRORS
ifdef GMSL_NO_WARNINGS
__gmsl_warning :=
else
__gmsl_warning = $(if $1,$(warning $(__gmsl_name): $1))
endif
ifdef GMSL_NO_ERRORS
__gmsl_error :=
else
__gmsl_error = $(if $1,$(error $(__gmsl_name): $1))
endif
# If GMSL_TRACE is enabled then calls to the library functions are
# traced to stdout using warning messages with their arguments
ifdef GMSL_TRACE
__gmsl_tr1 = $(warning $0('$1'))
__gmsl_tr2 = $(warning $0('$1','$2'))
__gmsl_tr3 = $(warning $0('$1','$2','$3'))
else
__gmsl_tr1 :=
__gmsl_tr2 :=
__gmsl_tr3 :=
endif
# See if spaces are valid in variable names (this was the case until
# GNU Make 3.82)
ifeq ($(MAKE_VERSION),3.82)
__gmsl_spaced_vars := $(false)
else
__gmsl_spaced_vars := $(true)
endif
# Figure out whether we have $(eval) or not (GNU Make 3.80 and above)
# if we do not then output a warning message, if we do then some
# functions will be enabled.
__gmsl_have_eval := $(false)
__gmsl_ignore := $(eval __gmsl_have_eval := $(true))
# If this is being run with Electric Cloud's emake then warn that
# their $(eval) support is incomplete in 1.x, 2.x, 3.x, 4.x and 5.0,
# 5.1, 5.2 and 5.3
ifdef ECLOUD_BUILD_ID
__gmsl_emake_major := $(word 1,$(subst ., ,$(EMAKE_VERSION)))
__gmsl_emake_minor := $(word 2,$(subst ., ,$(EMAKE_VERSION)))
ifneq ("$(findstring $(__gmsl_emake_major),1 2 3 4)$(findstring $(__gmsl_emake_major)$(__gmsl_emake_minor),50 51 52 53)","")
$(warning You are using a version of Electric Cloud's emake which has incomplete $$(eval) support)
__gmsl_have_eval := $(false)
endif
endif
# See if we have $(lastword) (GNU Make 3.81 and above)
__gmsl_have_lastword := $(lastword $(false) $(true))
# See if we have native or and and (GNU Make 3.81 and above)
__or_tt := /$(or $(true),$(true))/$(or $(true),$(false))/$(or $(false),$(true))/$(or $(false),$(false))/
__and_tt := /$(and $(true),$(true))/$(and $(true),$(false))/$(and $(false),$(true))/$(and $(false),$(false))/
__gmsl_have_or := $(if $(filter /T/T/T//,$(__or_tt)),$(true),$(false))
__gmsl_have_and := $(if $(filter /T////,$(__and_tt)),$(true),$(false))
ifneq ($(__gmsl_have_eval),$(true))
$(call __gmsl_warning,Your make version $(MAKE_VERSION) does not support $$$$(eval): some functions disabled)
endif
__gmsl_dollar := $$
__gmsl_hash := \#
# ----------------------------------------------------------------------------
# ----------------------------------------------------------------------------
# Function: gmsl_compatible
# Arguments: List containing the desired library version number (maj min rev)
# Returns: $(true) if this version of the library is compatible
# with the requested version number, otherwise $(false)
# ----------------------------------------------------------------------------
gmsl_compatible = $(strip \
$(if $(call gt,$(word 1,$1),$(word 1,$(gmsl_version))), \
$(false), \
$(if $(call lt,$(word 1,$1),$(word 1,$(gmsl_version))), \
$(true), \
$(if $(call gt,$(word 2,$1),$(word 2,$(gmsl_version))), \
$(false), \
$(if $(call lt,$(word 2,$1),$(word 2,$(gmsl_version))), \
$(true), \
$(call lte,$(word 3,$1),$(word 3,$(gmsl_version))))))))
# ###########################################################################
# LOGICAL OPERATORS
# ###########################################################################
# not is defined in gmsl
# ----------------------------------------------------------------------------
# Function: and
# Arguments: Two boolean values
# Returns: Returns $(true) if both of the booleans are true
# ----------------------------------------------------------------------------
ifneq ($(__gmsl_have_and),$(true))
and = $(__gmsl_tr2)$(if $1,$(if $2,$(true),$(false)),$(false))
endif
# ----------------------------------------------------------------------------
# Function: or
# Arguments: Two boolean values
# Returns: Returns $(true) if either of the booleans is true
# ----------------------------------------------------------------------------
ifneq ($(__gmsl_have_or),$(true))
or = $(__gmsl_tr2)$(if $1$2,$(true),$(false))
endif
# ----------------------------------------------------------------------------
# Function: xor
# Arguments: Two boolean values
# Returns: Returns $(true) if exactly one of the booleans is true
# ----------------------------------------------------------------------------
xor = $(__gmsl_tr2)$(if $1,$(if $2,$(false),$(true)),$(if $2,$(true),$(false)))
# ----------------------------------------------------------------------------
# Function: nand
# Arguments: Two boolean values
# Returns: Returns value of 'not and'
# ----------------------------------------------------------------------------
nand = $(__gmsl_tr2)$(if $1,$(if $2,$(false),$(true)),$(true))
# ----------------------------------------------------------------------------
# Function: nor
# Arguments: Two boolean values
# Returns: Returns value of 'not or'
# ----------------------------------------------------------------------------
nor = $(__gmsl_tr2)$(if $1$2,$(false),$(true))
# ----------------------------------------------------------------------------
# Function: xnor
# Arguments: Two boolean values
# Returns: Returns value of 'not xor'
# ----------------------------------------------------------------------------
xnor =$(__gmsl_tr2)$(if $1,$(if $2,$(true),$(false)),$(if $2,$(false),$(true)))
# ###########################################################################
# LIST MANIPULATION FUNCTIONS
# ###########################################################################
# ----------------------------------------------------------------------------
# Function: first (same as LISP's car, or head)
# Arguments: 1: A list
# Returns: Returns the first element of a list
# ----------------------------------------------------------------------------
first = $(__gmsl_tr1)$(firstword $1)
# ----------------------------------------------------------------------------
# Function: last
# Arguments: 1: A list
# Returns: Returns the last element of a list
# ----------------------------------------------------------------------------
ifeq ($(__gmsl_have_lastword),$(true))
last = $(__gmsl_tr1)$(lastword $1)
else
last = $(__gmsl_tr1)$(if $1,$(word $(words $1),$1))
endif
# ----------------------------------------------------------------------------
# Function: rest (same as LISP's cdr, or tail)
# Arguments: 1: A list
# Returns: Returns the list with the first element removed
# ----------------------------------------------------------------------------
rest = $(__gmsl_tr1)$(wordlist 2,$(words $1),$1)
# ----------------------------------------------------------------------------
# Function: chop
# Arguments: 1: A list
# Returns: Returns the list with the last element removed
# ----------------------------------------------------------------------------
chop = $(__gmsl_tr1)$(wordlist 2,$(words $1),x $1)
# ----------------------------------------------------------------------------
# Function: map
# Arguments: 1: Name of function to $(call) for each element of list
# 2: List to iterate over calling the function in 1
# Returns: The list after calling the function on each element
# ----------------------------------------------------------------------------
map = $(__gmsl_tr2)$(strip $(foreach a,$2,$(call $1,$a)))
# ----------------------------------------------------------------------------
# Function: pairmap
# Arguments: 1: Name of function to $(call) for each pair of elements
# 2: List to iterate over calling the function in 1
# 3: Second list to iterate over calling the function in 1
# Returns: The list after calling the function on each pair of elements
# ----------------------------------------------------------------------------
pairmap = $(strip $(__gmsl_tr3)\
$(if $2$3,$(call $1,$(call first,$2),$(call first,$3)) \
$(call pairmap,$1,$(call rest,$2),$(call rest,$3))))
# ----------------------------------------------------------------------------
# Function: leq
# Arguments: 1: A list to compare against...
# 2: ...this list
# Returns: Returns $(true) if the two lists are identical
# ----------------------------------------------------------------------------
leq = $(__gmsl_tr2)$(strip $(if $(call seq,$(words $1),$(words $2)), \
$(call __gmsl_list_equal,$1,$2),$(false)))
__gmsl_list_equal = $(if $(strip $1), \
$(if $(call seq,$(call first,$1),$(call first,$2)), \
$(call __gmsl_list_equal, \
$(call rest,$1), \
$(call rest,$2)), \
$(false)), \
$(true))
# ----------------------------------------------------------------------------
# Function: lne
# Arguments: 1: A list to compare against...
# 2: ...this list
# Returns: Returns $(true) if the two lists are different
# ----------------------------------------------------------------------------
lne = $(__gmsl_tr2)$(call not,$(call leq,$1,$2))
# ----------------------------------------------------------------------------
# Function: reverse
# Arguments: 1: A list to reverse
# Returns: The list with its elements in reverse order
# ----------------------------------------------------------------------------
reverse =$(__gmsl_tr1)$(strip $(if $1,$(call reverse,$(call rest,$1)) \
$(call first,$1)))
# ----------------------------------------------------------------------------
# Function: uniq
# Arguments: 1: A list from which to remove repeated elements
# Returns: The list with duplicate elements removed without reordering
# ----------------------------------------------------------------------------
uniq = $(strip $(__gmsl_tr1) $(if $1,$(firstword $1) \
$(call uniq,$(filter-out $(firstword $1),$1))))
# ----------------------------------------------------------------------------
# Function: length
# Arguments: 1: A list
# Returns: The number of elements in the list
# ----------------------------------------------------------------------------
length = $(__gmsl_tr1)$(words $1)
# ###########################################################################
# STRING MANIPULATION FUNCTIONS
# ###########################################################################
# Helper function that translates any GNU Make 'true' value (i.e. a
# non-empty string) to our $(true)
__gmsl_make_bool = $(if $(strip $1),$(true),$(false))
# ----------------------------------------------------------------------------
# Function: seq
# Arguments: 1: A string to compare against...
# 2: ...this string
# Returns: Returns $(true) if the two strings are identical
# ----------------------------------------------------------------------------
seq = $(__gmsl_tr2)$(if $(subst x$1,,x$2)$(subst x$2,,x$1),$(false),$(true))
# ----------------------------------------------------------------------------
# Function: sne
# Arguments: 1: A string to compare against...
# 2: ...this string
# Returns: Returns $(true) if the two strings are not the same
# ----------------------------------------------------------------------------
sne = $(__gmsl_tr2)$(call not,$(call seq,$1,$2))
# ----------------------------------------------------------------------------
# Function: split
# Arguments: 1: The character to split on
# 2: A string to split
# Returns: Splits a string into a list separated by spaces at the split
# character in the first argument
# ----------------------------------------------------------------------------
split = $(__gmsl_tr2)$(strip $(subst $1, ,$2))
# ----------------------------------------------------------------------------
# Function: merge
# Arguments: 1: The character to put between fields
# 2: A list to merge into a string
# Returns: Merges a list into a single string, list elements are separated
# by the character in the first argument
# ----------------------------------------------------------------------------
merge = $(__gmsl_tr2)$(strip $(if $2, \
$(if $(call seq,1,$(words $2)), \
$2,$(call first,$2)$1$(call merge,$1,$(call rest,$2)))))
ifdef __gmsl_have_eval
# ----------------------------------------------------------------------------
# Function: tr
# Arguments: 1: The list of characters to translate from
# 2: The list of characters to translate to
# 3: The text to translate
# Returns: Returns the text after translating characters
# ----------------------------------------------------------------------------
tr = $(strip $(__gmsl_tr3)$(call assert_no_dollar,$0,$1$2$3) \
$(eval __gmsl_t := $3) \
$(foreach c, \
$(join $(addsuffix :,$1),$2), \
$(eval __gmsl_t := \
$(subst $(word 1,$(subst :, ,$c)),$(word 2,$(subst :, ,$c)), \
$(__gmsl_t))))$(__gmsl_t))
# Common character classes for use with the tr function. Each of
# these is actually a variable declaration and must be wrapped with
# $() or ${} to be used.
[A-Z] := A B C D E F G H I J K L M N O P Q R S T U V W X Y Z #
[a-z] := a b c d e f g h i j k l m n o p q r s t u v w x y z #
[0-9] := 0 1 2 3 4 5 6 7 8 9 #
[A-F] := A B C D E F #
# ----------------------------------------------------------------------------
# Function: uc
# Arguments: 1: Text to upper case
# Returns: Returns the text in upper case
# ----------------------------------------------------------------------------
uc = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(call tr,$([a-z]),$([A-Z]),$1)
# ----------------------------------------------------------------------------
# Function: lc
# Arguments: 1: Text to lower case
# Returns: Returns the text in lower case
# ----------------------------------------------------------------------------
lc = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(call tr,$([A-Z]),$([a-z]),$1)
# ----------------------------------------------------------------------------
# Function: strlen
# Arguments: 1: A string
# Returns: Returns the length of the string
# ----------------------------------------------------------------------------
# This results in __gmsl_tab containing a tab
__gmsl_tab := #
__gmsl_characters := A B C D E F G H I J K L M N O P Q R S T U V W X Y Z
__gmsl_characters += a b c d e f g h i j k l m n o p q r s t u v w x y z
__gmsl_characters += 0 1 2 3 4 5 6 7 8 9
__gmsl_characters += ` ~ ! @ \# $$ % ^ & * ( ) - _ = +
__gmsl_characters += { } [ ] \ : ; ' " < > , . / ? |
# This results in __gmsl_space containing just a space
__gmsl_empty :=
__gmsl_space := $(__gmsl_empty) $(__gmsl_empty)
strlen = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(strip $(eval __temp := $(subst $(__gmsl_space),x,$1))$(foreach a,$(__gmsl_characters),$(eval __temp := $$(subst $$a,x,$(__temp))))$(eval __temp := $(subst x,x ,$(__temp)))$(words $(__temp)))
# This results in __gmsl_newline containing just a newline
define __gmsl_newline
endef
# ----------------------------------------------------------------------------
# Function: substr
# Arguments: 1: A string
# 2: Start position (first character is 1)
# 3: End position (inclusive)
# Returns: A substring.
# Note: The string in $1 must not contain a §
# ----------------------------------------------------------------------------
substr = $(if $2,$(__gmsl_tr3)$(call assert_no_dollar,$0,$1$2$3)$(strip $(eval __temp := $$(subst $$(__gmsl_space),§ ,$$1))$(foreach a,$(__gmsl_characters),$(eval __temp := $$(subst $$a,$$a$$(__gmsl_space),$(__temp))))$(eval __temp := $(wordlist $2,$3,$(__temp))))$(subst §,$(__gmsl_space),$(subst $(__gmsl_space),,$(__temp))))
endif # __gmsl_have_eval
# ###########################################################################
# SET MANIPULATION FUNCTIONS
# ###########################################################################
# Sets are represented by sorted, deduplicated lists. To create a set
# from a list use set_create, or start with the empty_set and
# set_insert individual elements
# This is the empty set
empty_set :=
# ----------------------------------------------------------------------------
# Function: set_create
# Arguments: 1: A list of set elements
# Returns: Returns the newly created set
# ----------------------------------------------------------------------------
set_create = $(__gmsl_tr1)$(sort $1)
# ----------------------------------------------------------------------------
# Function: set_insert
# Arguments: 1: A single element to add to a set
# 2: A set
# Returns: Returns the set with the element added
# ----------------------------------------------------------------------------
set_insert = $(__gmsl_tr2)$(sort $1 $2)
# ----------------------------------------------------------------------------
# Function: set_remove
# Arguments: 1: A single element to remove from a set
# 2: A set
# Returns: Returns the set with the element removed
# ----------------------------------------------------------------------------
set_remove = $(__gmsl_tr2)$(filter-out $1,$2)
# ----------------------------------------------------------------------------
# Function: set_is_member, set_is_not_member
# Arguments: 1: A single element
# 2: A set
# Returns: (set_is_member) Returns $(true) if the element is in the set
# (set_is_not_member) Returns $(false) if the element is in the set
# ----------------------------------------------------------------------------
set_is_member = $(__gmsl_tr2)$(if $(filter $1,$2),$(true),$(false))
set_is_not_member = $(__gmsl_tr2)$(if $(filter $1,$2),$(false),$(true))
# ----------------------------------------------------------------------------
# Function: set_union
# Arguments: 1: A set
# 2: Another set
# Returns: Returns the union of the two sets
# ----------------------------------------------------------------------------
set_union = $(__gmsl_tr2)$(sort $1 $2)
# ----------------------------------------------------------------------------
# Function: set_intersection
# Arguments: 1: A set
# 2: Another set
# Returns: Returns the intersection of the two sets
# ----------------------------------------------------------------------------
set_intersection = $(__gmsl_tr2)$(filter $1,$2)
# ----------------------------------------------------------------------------
# Function: set_is_subset
# Arguments: 1: A set
# 2: Another set
# Returns: Returns $(true) if the first set is a subset of the second
# ----------------------------------------------------------------------------
set_is_subset = $(__gmsl_tr2)$(call set_equal,$(call set_intersection,$1,$2),$1)
# ----------------------------------------------------------------------------
# Function: set_equal
# Arguments: 1: A set
# 2: Another set
# Returns: Returns $(true) if the two sets are identical
# ----------------------------------------------------------------------------
set_equal = $(__gmsl_tr2)$(call seq,$1,$2)
# ###########################################################################
# ARITHMETIC LIBRARY
# ###########################################################################
# Integers a represented by lists with the equivalent number of x's.
# For example the number 4 is x x x x.
# ----------------------------------------------------------------------------
# Function: int_decode
# Arguments: 1: A number of x's representation
# Returns: Returns the integer for human consumption that is represented
# by the string of x's
# ----------------------------------------------------------------------------
int_decode = $(__gmsl_tr1)$(if $1,$(if $(call seq,$(word 1,$1),x),$(words $1),$1),0)
# ----------------------------------------------------------------------------
# Function: int_encode
# Arguments: 1: A number in human-readable integer form
# Returns: Returns the integer encoded as a string of x's
# ----------------------------------------------------------------------------
__int_encode = $(if $1,$(if $(call seq,$(words $(wordlist 1,$1,$2)),$1),$(wordlist 1,$1,$2),$(call __int_encode,$1,$(if $2,$2 $2,x))))
__strip_leading_zero = $(if $1,$(if $(call seq,$(patsubst 0%,%,$1),$1),$1,$(call __strip_leading_zero,$(patsubst 0%,%,$1))),0)
int_encode = $(__gmsl_tr1)$(call __int_encode,$(call __strip_leading_zero,$1))
# The arithmetic library functions come in two forms: one form of each
# function takes integers as arguments and the other form takes the
# encoded form (x's created by a call to int_encode). For example,
# there are two plus functions:
#
# plus Called with integer arguments and returns an integer
# int_plus Called with encoded arguments and returns an encoded result
#
# plus will be slower than int_plus because its arguments and result
# have to be translated between the x's format and integers. If doing
# a complex calculation use the int_* forms with a single encoding of
# inputs and single decoding of the output. For simple calculations
# the direct forms can be used.
# Helper function used to wrap an int_* function into a function that
# takes a pair of integers, perhaps a function and returns an integer
# result
__gmsl_int_wrap = $(call int_decode,$(call $1,$(call int_encode,$2),$(call int_encode,$3)))
__gmsl_int_wrap1 = $(call int_decode,$(call $1,$(call int_encode,$2)))
__gmsl_int_wrap2 = $(call $1,$(call int_encode,$2),$(call int_encode,$3))
# ----------------------------------------------------------------------------
# Function: int_plus
# Arguments: 1: A number in x's representation
# 2: Another number in x's represntation
# Returns: Returns the sum of the two numbers in x's representation
# ----------------------------------------------------------------------------
int_plus = $(strip $(__gmsl_tr2)$1 $2)
# ----------------------------------------------------------------------------
# Function: plus (wrapped version of int_plus)
# Arguments: 1: An integer
# 2: Another integer
# Returns: Returns the sum of the two integers
# ----------------------------------------------------------------------------
plus = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_plus,$1,$2)
# ----------------------------------------------------------------------------
# Function: int_subtract
# Arguments: 1: A number in x's representation
# 2: Another number in x's represntation
# Returns: Returns the difference of the two numbers in x's representation,
# or outputs an error on a numeric underflow
# ----------------------------------------------------------------------------
int_subtract = $(strip $(__gmsl_tr2)$(if $(call int_gte,$1,$2), \
$(filter-out xx,$(join $1,$2)), \
$(call __gmsl_warning,Subtraction underflow)))
# ----------------------------------------------------------------------------
# Function: subtract (wrapped version of int_subtract)
# Arguments: 1: An integer
# 2: Another integer
# Returns: Returns the difference of the two integers,
# or outputs an error on a numeric underflow
# ----------------------------------------------------------------------------
subtract = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_subtract,$1,$2)
# ----------------------------------------------------------------------------
# Function: int_multiply
# Arguments: 1: A number in x's representation
# 2: Another number in x's represntation
# Returns: Returns the product of the two numbers in x's representation
# ----------------------------------------------------------------------------
int_multiply = $(strip $(__gmsl_tr2)$(foreach a,$1,$2))
# ----------------------------------------------------------------------------
# Function: multiply (wrapped version of int_multiply)
# Arguments: 1: An integer
# 2: Another integer
# Returns: Returns the product of the two integers
# ----------------------------------------------------------------------------
multiply = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_multiply,$1,$2)
# ----------------------------------------------------------------------------
# Function: int_divide
# Arguments: 1: A number in x's representation
# 2: Another number in x's represntation
# Returns: Returns the result of integer division of argument 1 divided
# by argument 2 in x's representation
# ----------------------------------------------------------------------------
int_divide = $(__gmsl_tr2)$(strip $(if $1,$(if $2, \
$(subst M,x,$(filter-out x,$(subst $2,M,$1))), \
$(call __gmsl_error,Division by zero))))
# ----------------------------------------------------------------------------
# Function: divide (wrapped version of int_divide)
# Arguments: 1: An integer
# 2: Another integer
# Returns: Returns the integer division of the first argument by the second
# ----------------------------------------------------------------------------
divide = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_divide,$1,$2)
# ----------------------------------------------------------------------------
# Function: int_modulo
# Arguments: 1: A number in x's representation
# 2: Another number in x's represntation
# Returns: Returns the remainder of integer division of argument 1 divided
# by argument 2 in x's representation
# ----------------------------------------------------------------------------
int_modulo = $(__gmsl_tr2)$(strip $(if $1,$(if $2, \
$(filter-out M,$(subst $2,M,$1)), \
$(call __gmsl_error,Division by zero))))
# ----------------------------------------------------------------------------
# Function: modulo (wrapped version of int_modulo)
# Arguments: 1: An integer
# 2: Another integer
# Returns: Returns the remainder of integer division of the first argument
# by the second
# ----------------------------------------------------------------------------
modulo = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_modulo,$1,$2)
# ----------------------------------------------------------------------------
# Function: int_max, int_min
# Arguments: 1: A number in x's representation
# 2: Another number in x's represntation
# Returns: Returns the maximum or minimum of its arguments in x's
# representation
# ----------------------------------------------------------------------------
int_max = $(__gmsl_tr2)$(subst xx,x,$(join $1,$2))
int_min = $(__gmsl_tr2)$(subst xx,x,$(filter xx,$(join $1,$2)))
# ----------------------------------------------------------------------------
# Function: max, min
# Arguments: 1: An integer
# 2: Another integer
# Returns: Returns the maximum or minimum of its integer arguments
# ----------------------------------------------------------------------------
max = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_max,$1,$2)
min = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_min,$1,$2)
# ----------------------------------------------------------------------------
# Function: int_gt, int_gte, int_lt, int_lte, int_eq, int_ne
# Arguments: Two x's representation numbers to be compared
# Returns: $(true) or $(false)
#
# int_gt First argument greater than second argument
# int_gte First argument greater than or equal to second argument
# int_lt First argument less than second argument
# int_lte First argument less than or equal to second argument
# int_eq First argument is numerically equal to the second argument
# int_ne First argument is not numerically equal to the second argument
# ----------------------------------------------------------------------------
int_gt = $(__gmsl_tr2)$(call __gmsl_make_bool, \
$(filter-out $(words $2), \
$(words $(call int_max,$1,$2))))
int_gte = $(__gmsl_tr2)$(call __gmsl_make_bool, \
$(call int_gt,$1,$2)$(call int_eq,$1,$2))
int_lt = $(__gmsl_tr2)$(call __gmsl_make_bool, \
$(filter-out $(words $1), \
$(words $(call int_max,$1,$2))))
int_lte = $(__gmsl_tr2)$(call __gmsl_make_bool, \
$(call int_lt,$1,$2)$(call int_eq,$1,$2))
int_eq = $(__gmsl_tr2)$(call __gmsl_make_bool, \
$(filter $(words $1),$(words $2)))
int_ne = $(__gmsl_tr2)$(call __gmsl_make_bool, \
$(filter-out $(words $1),$(words $2)))
# ----------------------------------------------------------------------------
# Function: gt, gte, lt, lte, eq, ne
# Arguments: Two integers to be compared
# Returns: $(true) or $(false)
#
# gt First argument greater than second argument
# gte First argument greater than or equal to second argument
# lt First argument less than second argument
# lte First argument less than or equal to second argument
# eq First argument is numerically equal to the second argument
# ne First argument is not numerically equal to the second argument
# ----------------------------------------------------------------------------
gt = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_gt,$1,$2)
gte = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_gte,$1,$2)
lt = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_lt,$1,$2)
lte = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_lte,$1,$2)
eq = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_eq,$1,$2)
ne = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_ne,$1,$2)
# increment adds 1 to its argument, decrement subtracts 1. Note that
# decrement does not range check and hence will not underflow, but
# will incorrectly say that 0 - 1 = 0
# ----------------------------------------------------------------------------
# Function: int_inc
# Arguments: 1: A number in x's representation
# Returns: The number incremented by 1 in x's representation
# ----------------------------------------------------------------------------
int_inc = $(strip $(__gmsl_tr1)$1 x)
# ----------------------------------------------------------------------------
# Function: inc
# Arguments: 1: An integer
# Returns: The argument incremented by 1
# ----------------------------------------------------------------------------
inc = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_inc,$1)
# ----------------------------------------------------------------------------
# Function: int_dec
# Arguments: 1: A number in x's representation
# Returns: The number decremented by 1 in x's representation
# ----------------------------------------------------------------------------
int_dec = $(__gmsl_tr1)$(strip \
$(if $(call sne,0,$(words $1)), \
$(wordlist 2,$(words $1),$1)))
# ----------------------------------------------------------------------------
# Function: dec
# Arguments: 1: An integer
# Returns: The argument decremented by 1
# ----------------------------------------------------------------------------
dec = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_dec,$1)
# double doubles its argument, and halve halves it
# ----------------------------------------------------------------------------
# Function: int_double
# Arguments: 1: A number in x's representation
# Returns: The number doubled (i.e. * 2) and returned in x's representation
# ----------------------------------------------------------------------------
int_double = $(strip $(__gmsl_tr1)$1 $1)
# ----------------------------------------------------------------------------
# Function: double
# Arguments: 1: An integer
# Returns: The integer times 2
# ----------------------------------------------------------------------------
double = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_double,$1)
# ----------------------------------------------------------------------------
# Function: int_halve
# Arguments: 1: A number in x's representation
# Returns: The number halved (i.e. / 2) and returned in x's representation
# ----------------------------------------------------------------------------
int_halve = $(__gmsl_tr1)$(strip $(subst xx,x,$(filter-out xy x y, \
$(join $1,$(foreach a,$1,y x)))))
# ----------------------------------------------------------------------------
# Function: halve
# Arguments: 1: An integer
# Returns: The integer divided by 2
# ----------------------------------------------------------------------------
halve = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_halve,$1)
# ----------------------------------------------------------------------------
# Function: sequence
# Arguments: 1: An integer
# 2: An integer
# Returns: The sequence [arg1, arg2] of integers if arg1 < arg2 or
# [arg2, arg1] if arg2 > arg1. If arg1 == arg1 return [arg1]
# ----------------------------------------------------------------------------
sequence = $(__gmsl_tr2)$(strip $(if $(call lte,$1,$2), \
$(call __gmsl_sequence_up,$1,$2), \
$(call __gmsl_sequence_dn,$2,$1)))
__gmsl_sequence_up = $(if $(call seq,$1,$2),$1,$1 $(call __gmsl_sequence_up,$(call inc,$1),$2))
__gmsl_sequence_dn = $(if $(call seq,$1,$2),$1,$2 $(call __gmsl_sequence_dn,$1,$(call dec,$2)))
# ----------------------------------------------------------------------------
# Function: dec2hex, dec2bin, dec2oct
# Arguments: 1: An integer
# Returns: The decimal argument converted to hexadecimal, binary or
# octal
# ----------------------------------------------------------------------------
__gmsl_digit = $(subst 15,f,$(subst 14,e,$(subst 13,d,$(subst 12,c,$(subst 11,b,$(subst 10,a,$1))))))
dec2hex = $(call __gmsl_dec2base,$(call int_encode,$1),$(call int_encode,16))
dec2bin = $(call __gmsl_dec2base,$(call int_encode,$1),$(call int_encode,2))
dec2oct = $(call __gmsl_dec2base,$(call int_encode,$1),$(call int_encode,8))
__gmsl_base_divide = $(subst $2,X ,$1)
__gmsl_q = $(strip $(filter X,$1))
__gmsl_r = $(words $(filter x,$1))
__gmsl_dec2base = $(eval __gmsl_temp := $(call __gmsl_base_divide,$1,$2))$(call __gmsl_dec2base_,$(call __gmsl_q,$(__gmsl_temp)),$(call __gmsl_r,$(__gmsl_temp)),$2)
__gmsl_dec2base_ = $(if $1,$(call __gmsl_dec2base,$(subst X,x,$1),$3))$(call __gmsl_digit,$2)
ifdef __gmsl_have_eval
# ###########################################################################
# ASSOCIATIVE ARRAYS
# ###########################################################################
# Magic string that is very unlikely to appear in a key or value
__gmsl_aa_magic := faf192c8efbc25c27992c5bc5add390393d583c6
# ----------------------------------------------------------------------------
# Function: set
# Arguments: 1: Name of associative array
# 2: The key value to associate
# 3: The value associated with the key
# Returns: Nothing
# ----------------------------------------------------------------------------
set = $(__gmsl_tr3)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2$3)$(eval __gmsl_aa_$1_$(__gmsl_aa_magic)_$2_gmsl_aa_$1 := $3)
# Only used internally by memoize function
__gmsl_set = $(call set,$1,$2,$3)$3
# ----------------------------------------------------------------------------
# Function: get
# Arguments: 1: Name of associative array
# 2: The key to retrieve
# Returns: The value stored in the array for that key
# ----------------------------------------------------------------------------
get = $(strip $(__gmsl_tr2)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2)$(__gmsl_aa_$1_$(__gmsl_aa_magic)_$2_gmsl_aa_$1))
# ----------------------------------------------------------------------------
# Function: keys
# Arguments: 1: Name of associative array
# Returns: Returns a list of all defined keys in the array
# ----------------------------------------------------------------------------
keys = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(sort $(patsubst __gmsl_aa_$1_$(__gmsl_aa_magic)_%_gmsl_aa_$1,%, \
$(filter __gmsl_aa_$1_$(__gmsl_aa_magic)_%_gmsl_aa_$1,$(.VARIABLES))))
# ----------------------------------------------------------------------------
# Function: defined
# Arguments: 1: Name of associative array
# 2: The key to test
# Returns: Returns true if the key is defined (i.e. not empty)
# ----------------------------------------------------------------------------
defined = $(__gmsl_tr2)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2)$(call sne,$(call get,$1,$2),)
endif # __gmsl_have_eval
ifdef __gmsl_have_eval
# ###########################################################################
# NAMED STACKS
# ###########################################################################
# ----------------------------------------------------------------------------
# Function: push
# Arguments: 1: Name of stack
# 2: Value to push onto the top of the stack (must not contain
# a space)
# Returns: None
# ----------------------------------------------------------------------------
push = $(__gmsl_tr2)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2)$(eval __gmsl_stack_$1 := $2 $(if $(filter-out undefined,\
$(origin __gmsl_stack_$1)),$(__gmsl_stack_$1)))
# ----------------------------------------------------------------------------
# Function: pop
# Arguments: 1: Name of stack
# Returns: Top element from the stack after removing it
# ----------------------------------------------------------------------------
pop = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(strip $(if $(filter-out undefined,$(origin __gmsl_stack_$1)), \
$(call first,$(__gmsl_stack_$1)) \
$(eval __gmsl_stack_$1 := $(call rest,$(__gmsl_stack_$1)))))
# ----------------------------------------------------------------------------
# Function: peek
# Arguments: 1: Name of stack
# Returns: Top element from the stack without removing it
# ----------------------------------------------------------------------------
peek = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(call first,$(__gmsl_stack_$1))
# ----------------------------------------------------------------------------
# Function: depth
# Arguments: 1: Name of stack
# Returns: Number of items on the stack
# ----------------------------------------------------------------------------
depth = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(words $(__gmsl_stack_$1))
endif # __gmsl_have_eval
ifdef __gmsl_have_eval
# ###########################################################################
# STRING CACHE
# ###########################################################################
# ----------------------------------------------------------------------------
# Function: memoize
# Arguments: 1. Name of the function to be called if the string
# has not been previously seen
# 2. A string
# Returns: Returns the result of a memo function (which the user must
# define) on the passed in string and remembers the result.
#
# Example: Set memo = $(shell echo "$1" | md5sum) to make a cache
# of MD5 hashes of strings. $(call memoize,memo,foo bar baz)
# ----------------------------------------------------------------------------
__gmsl_memoize = $(subst $(__gmsl_space),§,$1)cc2af1bb7c4482f2ba75e338b963d3e7$(subst $(__gmsl_space),§,$2)
memoize = $(__gmsl_tr2)$(strip $(if $(call defined,__gmsl_m,$(__gmsl_memoize)),\
$(call get,__gmsl_m,$(__gmsl_memoize)), \
$(call __gmsl_set,__gmsl_m,$(__gmsl_memoize),$(call $1,$2))))
endif # __gmsl_have_eval
# ###########################################################################
# DEBUGGING FACILITIES
# ###########################################################################
# ----------------------------------------------------------------------------
# Target: gmsl-echo-%
# Arguments: The % should be replaced by the name of a variable that you
# wish to print out.
# Action: Echos the value of the variable that matches the %.
# For example, 'make gmsl-echo-SHELL' will output the value of
# the SHELL variable.
# ----------------------------------------------------------------------------
gmsl-echo-%: ; @echo $($*)
# ----------------------------------------------------------------------------
# Target: gmsl-print-%
# Arguments: The % should be replaced by the name of a variable that you
# wish to print out.
# Action: Echos the name of the variable that matches the % and its value.
# For example, 'make gmsl-print-SHELL' will output the value of
# the SHELL variable
# ----------------------------------------------------------------------------
gmsl-print-%: ; @echo $* = $($*)
# ----------------------------------------------------------------------------
# Function: assert
# Arguments: 1: A boolean that must be true or the assertion will fail
# 2: The message to print with the assertion
# Returns: None
# ----------------------------------------------------------------------------
assert = $(if $2,$(if $1,,$(call __gmsl_error,Assertion failure: $2)))
# ----------------------------------------------------------------------------
# Function: assert_exists
# Arguments: 1: Name of file that must exist, if it is missing an assertion
# will be generated
# Returns: None
# ----------------------------------------------------------------------------
assert_exists = $(if $0,$(call assert,$(wildcard $1),file '$1' missing))
# ----------------------------------------------------------------------------
# Function: assert_no_dollar
# Arguments: 1: Name of a function being executd
# 2: Arguments to check
# Returns: None
# ----------------------------------------------------------------------------
assert_no_dollar = $(call __gmsl_tr2)$(call assert,$(call not,$(findstring $(__gmsl_dollar),$2)),$1 called with a dollar sign in argument)
# ----------------------------------------------------------------------------
# Function: assert_no_space
# Arguments: 1: Name of a function being executd
# 2: Arguments to check
# Returns: None
# ----------------------------------------------------------------------------
ifeq ($(__gmsl_spaced_vars),$(false))
assert_no_space = $(call assert,$(call not,$(findstring $(__gmsl_aa_magic),$(subst $(__gmsl_space),$(__gmsl_aa_magic),$2))),$1 called with a space in argument)
else
assert_no_space =
endif

85
mk/gmsl
View file

@ -1,85 +0,0 @@
# ----------------------------------------------------------------------------
#
# GNU Make Standard Library (GMSL)
#
# A library of functions to be used with GNU Make's $(call) that
# provides functionality not available in standard GNU Make.
#
# Copyright (c) 2005-2022 John Graham-Cumming
#
# This file is part of GMSL
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# Neither the name of the John Graham-Cumming nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# ----------------------------------------------------------------------------
# Determine if the library has already been included and if so don't
# bother including it again
ifndef __gmsl_included
# Standard definitions for true and false. true is any non-empty
# string, false is an empty string. These are intended for use with
# $(if).
true := T
false :=
# ----------------------------------------------------------------------------
# Function: not
# Arguments: 1: A boolean value
# Returns: Returns the opposite of the arg. (true -> false, false -> true)
# ----------------------------------------------------------------------------
not = $(if $1,$(false),$(true))
# Prevent reinclusion of the library
__gmsl_included := $(true)
# Try to determine where this file is located. If the caller did
# include /foo/gmsl then extract the /foo/ so that __gmsl gets
# included transparently
__gmsl_root :=
ifneq ($(MAKEFILE_LIST),)
__gmsl_root := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST))
# If there are any spaces in the path in __gmsl_root then give up
ifeq (1,$(words $(__gmsl_root)))
__gmsl_root := $(patsubst %gmsl,%,$(__gmsl_root))
endif
endif
include $(__gmsl_root)__gmsl
endif # __gmsl_included

733
mk/gmsl.html
View file

@ -1,733 +0,0 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html><head>
<meta content="text/html; charset=ISO-8859-1" http-equiv="content-type">
<title>GNU Make Standard Library</title></head>
<body>
<h1>GNU Make Standard Library</h1>
The GNU Make Standard Library (GMSL) is a collection of functions
implemented using native GNU Make functionality that provide list and
string manipulation, integer arithmetic, associative arrays, stacks,
and debugging facilities.&nbsp; The GMSL is released under the BSD License.<br>
<br>
<a href="https://github.com/jgrahamc/gmsl/">[Project Page]</a>
<a href="https://github.com/jgrahamc/gmsl/releases/">[Releases]</a>
<br>
<h2>Using GMSL</h2>
The two files needed are <span style="font-family: monospace;">gmsl</span>
and <span style="font-family: monospace;">__gmsl</span>.&nbsp; To
include the GMSL in your Makefile do<br>
<pre style="margin-left: 40px;">include gmsl</pre>
<span style="font-family: monospace;">gmsl</span> automatically includes<span style="font-family: monospace;"> __gmsl</span>.&nbsp; To check that
you have the right version of <span style="font-family: monospace;">gmsl</span>
use the <span style="font-family: monospace;">gmsl_compatible</span>
function (see
below). The current version is <span style="font-family: monospace;">1 2 0</span>.<br>
<br>
The GMSL package also includes a test suite for GMSL.&nbsp; Just run <span style="font-family: monospace;">make -f gmsl-tests</span>.<br>
<h2>Logical Operators</h2>GMSL has boolean $(true) (a non-empty string)
and $(false) (an empty string).&nbsp; The following operators can be
used with those variables.<br>
<br>
<hr style="width: 100%; height: 2px;"><span style="font-weight: bold;">not</span><br>
<br>
<span style="font-family: monospace;">Arguments: A boolean value</span><br style="font-family: monospace;">
<span style="font-family: monospace;">Returns:&nbsp;&nbsp; Returns $(true) if the boolean is $(false) and vice versa</span><br style="font-family: monospace;">
<hr style="width: 100%; height: 2px; font-family: monospace;"><span style="font-weight: bold;"></span><span style="font-weight: bold;">and</span><br>
<br>
<span style="font-family: monospace;">Arguments: Two boolean values</span><br style="font-family: monospace;">
<span style="font-family: monospace;">Returns:&nbsp;&nbsp; Returns $(true) if both of the booleans are true</span><br style="font-family: monospace;">
<hr style="width: 100%; height: 2px; font-family: monospace;"><span style="font-weight: bold;">or</span><br>
<br>
<span style="font-family: monospace;">Arguments: Two boolean values</span><br style="font-family: monospace;">
<span style="font-family: monospace;">Returns:&nbsp;&nbsp; Returns $(true) if either of the booleans is true</span><br style="font-family: monospace;">
<hr style="width: 100%; height: 2px; font-family: monospace;"><span style="font-weight: bold;">xor</span><br style="font-weight: bold;">
<br>
<span style="font-family: monospace;">Arguments: Two boolean values</span><br style="font-family: monospace;">
<span style="font-family: monospace;">Returns:&nbsp;&nbsp; Returns $(true) if exactly one of the booleans is true</span><br style="font-family: monospace;">
<hr style="width: 100%; height: 2px; font-family: monospace;"><span style="font-weight: bold;">nand</span><br>
<br>
<span style="font-family: monospace;">Arguments: Two boolean values</span><br style="font-family: monospace;">
<span style="font-family: monospace;">Returns:&nbsp;&nbsp; Returns value of 'not and'</span><br style="font-family: monospace;">
<hr style="width: 100%; height: 2px; font-family: monospace;"><span style="font-weight: bold;">nor</span><br>
<br>
<span style="font-family: monospace;">Arguments: Two boolean values</span><br style="font-family: monospace;">
<span style="font-family: monospace;">Returns:&nbsp;&nbsp; Returns value of 'not or'</span><br style="font-family: monospace;">
<hr style="width: 100%; height: 2px; font-family: monospace;"><span style="font-weight: bold;">xnor</span><br>
<br>
<span style="font-family: monospace;">Arguments: Two boolean values</span><br style="font-family: monospace;">
<span style="font-family: monospace;">Returns:&nbsp;&nbsp; Returns value of 'not xor'</span><br style="font-family: monospace;">
<hr style="width: 100%; height: 2px; font-family: monospace;">
<h2>List Manipulation Functions</h2>
&nbsp;A list is a string of characters; the list separator is a space.<br>
<br>
<hr style="width: 100%; height: 2px;"><b>first</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A list<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the first element of a list<br>
</span>
<hr><b>last</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A list<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the last element of a list<br>
</span>
<hr><b>rest</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A list<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the list with the first element
removed<br>
</span>
<hr><b>chop</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A list<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the list with the last element removed<br>
</span>
<hr><b>map</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of function to
$(call) for each element of list<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: List to
iterate over calling the function in 1<br>
Returns:&nbsp;&nbsp;&nbsp;The list after calling the function on each
element<br>
</span>
<hr><b>pairmap</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of function to
$(call) for each pair of elements<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: List to
iterate over calling the function in 1<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 3: Second
list to iterate over calling the function in 1<br>
Returns:&nbsp;&nbsp;&nbsp;The list after calling the function on each
pair of elements<br>
</span>
<hr><b>leq</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A list to compare
against...<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: ...this
list<br>
Returns:&nbsp;&nbsp;&nbsp;Returns $(true) if the two lists are identical<br>
</span>
<hr><b>lne</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A list to compare
against...<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: ...this
list<br>
Returns:&nbsp;&nbsp;&nbsp;Returns $(true) if the two lists are different<br>
</span>
<hr><b>reverse</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A list to reverse<br>
Returns:&nbsp;&nbsp;&nbsp;The list with its elements in reverse order<br>
</span>
<hr><b>uniq</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A list to deduplicate<br>
Returns:&nbsp;&nbsp;&nbsp;The list with elements in order without duplicates<br>
</span>
<hr><b>length</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A list<br>
Returns:&nbsp;&nbsp;&nbsp;The number of elements in the list<br>
</span>
<hr style="width: 100%; height: 2px;"><span style="font-family: monospace;"></span>
<h2>String Manipulation Functions</h2>
A string is any sequence of characters.<br>
<br>
<hr style="width: 100%; height: 2px;"><b>seq</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A string to compare
against...<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: ...this
string<br>
Returns:&nbsp;&nbsp;&nbsp;Returns $(true) if the two strings are
identical<br>
</span>
<hr><b>sne</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A string to compare
against...<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: ...this
string<br>
Returns:&nbsp;&nbsp;&nbsp;Returns $(true) if the two strings are not
the same<br>
</span>
<hr><b>strlen</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A string<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the length of the string<br>
</span>
<hr><b>substr</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A string<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Start offset (first character is 1)<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 3: Ending offset (inclusive)<br>Returns:&nbsp;&nbsp;&nbsp;Returns a substring<br>
</span>
<hr><b>split</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: The character to
split on<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: A
string to split<br>
Returns:&nbsp;&nbsp;&nbsp;Splits a string into a list separated by
spaces at the split<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; character
in the first argument<br>
</span>
<hr><b>merge</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: The character to
put between fields<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: A list
to merge into a string<br>
Returns:&nbsp;&nbsp;&nbsp;Merges a list into a single string, list
elements are separated<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; by the
character in the first argument<br>
</span>
<hr><b>tr</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: The list of
characters to translate from <br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: The
list of characters to translate to<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 3: The
text to translate<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the text after translating characters<br>
</span>
<hr><b>uc</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Text to upper case<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the text in upper case<br>
</span>
<hr><b>lc</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Text to lower case<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the text in lower case<br>
</span>
<hr style="width: 100%; height: 2px;"><span style="font-family: monospace;"></span>
<h2>Set Manipulation Functions</h2>
Sets are represented by sorted, deduplicated lists. To create a set
from a list use <span style="font-family:
monospace;">set_create</span>, or start with the <span
style="font-family: monospace;">empty_set</span> and <span
style="font-family: monospace;">set_insert</span> individual elements.
The empty set is defined as <span style="font-family:
monospace;">empty_set</span>.<p>
<hr><b>set_create</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A list of set elements<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the newly created set<br>
</span>
<hr><b>set_insert</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A single element to add to a set<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;2: A set<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the set with the element added<br>
</span>
<hr><b>set_remove</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A single element to remove from a set<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;2: A set<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the set with the element removed<br>
</span>
<hr><b>set_is_member</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A single element<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;2: A set<br>
Returns:&nbsp;&nbsp;&nbsp;Returns $(true) if the element is in the set<br>
</span>
<hr><b>set_is_not_member</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A single element<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;2: A set<br>
Returns:&nbsp;&nbsp;&nbsp;Returns $(false) if the element is in the set<br>
</span>
<hr><b>set_union</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A set<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;2: Another set<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the union of the two sets<br>
</span>
<hr><b>set_intersection</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A set<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;2: Another set<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the intersection of the two sets<br>
</span>
<hr><b>set_is_subset</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A set<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;2: Another set<br>
Returns:&nbsp;&nbsp;&nbsp;Returns $(true) if the first set is a subset of the second<br>
</span>
<hr><b>set_equal</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A set<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;2: Another set<br>
Returns:&nbsp;&nbsp;&nbsp;Returns $(true) if the two sets are identical<br>
</span>
<hr style="width: 100%; height: 2px;"><span style="font-family: monospace;"></span>
<h2>Integer Arithmetic Functions</h2>
Integers are represented by lists with the equivalent number of
x's.&nbsp; For example the number 4 is x x x x.&nbsp; The maximum
integer that the library can handle as <span style="font-style: italic;">input</span> (i.e. as the argument to a
call to <span style="font-family: monospace;">int_encode</span>) is
65536. There is no limit on integer size for internal computations or
output.<br>
<br>
The arithmetic library functions come in two forms: one form of each
function takes integers as arguments and the other form takes the
encoded form (x's created by a call to <span style="font-family: monospace;">int_encode</span>).&nbsp; For example,
there are two plus functions: <span style="font-family: monospace;">plus</span>
(called with integer arguments and returns an integer) and <span style="font-family: monospace;">int_plus</span> (called with encoded
arguments and returns an encoded result).<br>
<br>
<span style="font-family: monospace;">plus</span> will be slower than <span style="font-family: monospace;">int_plus</span> because its arguments
and result have to be translated between the x's format and
integers.&nbsp; If doing a complex calculation use the <span style="font-family: monospace;">int_*</span> forms with a single
encoding of inputs and single decoding of the output.&nbsp; For simple
calculations the direct forms can be used.<br>
<br>
<hr style="width: 100%; height: 2px;"><b>int_decode</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number of x's
representation<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the integer for human consumption
that is represented<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; by the
string of x's<br>
</span>
<hr><b>int_encode</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number in
human-readable integer form<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the integer encoded as a string of x's<br>
</span>
<hr><b>int_plus</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number in x's
representation<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
number in x's represntation<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the sum of the two numbers in x's
representation<br>
</span>
<hr><b>plus (wrapped version of int_plus)</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
integer<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the sum of the two integers<br>
</span>
<hr><b>int_subtract</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number in x's
representation<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
number in x's represntation<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the difference of the two numbers in
x's representation,<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; or outputs
an error on a numeric underflow<br>
</span>
<hr><b>subtract (wrapped version of int_subtract)</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
integer<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the difference of the two integers,<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; or outputs
an error on a numeric underflow<br>
</span>
<hr><b>int_multiply</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number in x's
representation<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
number in x's represntation<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the product of the two numbers in x's
representation<br>
</span>
<hr><b>multiply (wrapped version of int_multiply)</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
integer<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the product of the two integers<br>
</span>
<hr><b>int_divide</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number in x's
representation<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
number in x's represntation<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the result of integer division of
argument 1 divided<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; by
argument 2 in x's representation<br>
</span>
<hr><b>divide (wrapped version of int_divide)</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
integer<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the integer division of the first
argument by the second<br>
</span>
<hr><b>int_modulo</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number in x's
representation<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
number in x's represntation<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the remainder of integer division of
argument 1 divided<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; by
argument 2 in x's representation<br>
</span>
<hr><b>modulo (wrapped version of int_modulo)</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
integer<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the remainder of integer division of the first
argument by the second<br>
</span>
<hr><b>int_max, int_min</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number in x's
representation<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
number in x's represntation<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the maximum or minimum of its
arguments in x's<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
representation<br>
</span>
<hr><b>max, min</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
integer<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the maximum or minimum of its integer
arguments<br>
</span>
<hr><b>int_gt, int_gte, int_lt, int_lte, int_eq, int_ne</b><br>
<br>
<span style="font-family: monospace;">Arguments: Two x's representation
numbers to be compared<br>
Returns:&nbsp;&nbsp;&nbsp;$(true) or $(false)<br>
<br>
int_gt First argument greater than second argument<br>
int_gte First argument greater than or equal to second argument<br>
int_lt First argument less than second argument <br>
int_lte First argument less than or equal to second argument<br>
int_eq First argument is numerically equal to the second argument<br>
int_ne First argument is not numerically equal to the second argument<br>
</span>
<hr><b>gt, gte, lt, lte, eq, ne</b><br>
<br>
<span style="font-family: monospace;">Arguments: Two integers to be
compared<br>
Returns:&nbsp;&nbsp;&nbsp;$(true) or $(false)<br>
<br>
gt First argument greater than second argument<br>
gte First argument greater than or equal to second argument<br>
lt First argument less than second argument <br>
lte First argument less than or equal to second argument<br>
eq First argument is numerically equal to the second argument<br>
ne First argument is not numerically equal to the second argument<br>
</span>
increment adds 1 to its argument, decrement subtracts 1. Note that<br>
decrement does not range check and hence will not underflow, but<br>
will incorrectly say that 0 - 1 = 0<br>
<hr><b>int_inc</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number in x's
representation<br>
Returns:&nbsp;&nbsp;&nbsp;The number incremented by 1 in x's
representation<br>
</span>
<hr><b>inc</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
Returns:&nbsp;&nbsp;&nbsp;The argument incremented by 1<br>
</span>
<hr><b>int_dec</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number in x's
representation<br>
Returns:&nbsp;&nbsp;&nbsp;The number decremented by 1 in x's
representation<br>
</span>
<hr><b>dec</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
Returns:&nbsp;&nbsp;&nbsp;The argument decremented by 1<br>
</span>
<hr><b>int_double</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number in x's
representation<br>
Returns:&nbsp;&nbsp;&nbsp;The number doubled (i.e. * 2) and returned in
x's representation<br>
</span>
<hr><b>double</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
Returns:&nbsp;&nbsp;&nbsp;The integer times 2<br>
</span>
<hr><b>int_halve</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number in x's
representation<br>
Returns:&nbsp;&nbsp;&nbsp;The number halved (i.e. / 2) and returned in
x's representation<br>
</span>
<hr><b>halve</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
Returns:&nbsp;&nbsp;&nbsp;The integer divided by 2<br>
</span>
<hr><b>sequence</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: An integer<br>
Returns:&nbsp;&nbsp;&nbsp;The sequence [arg1 arg2] if arg1 >= arg2 or [arg2 arg1] if arg2 > arg1<br>
</span>
<hr><b>dec2hex, dec2bin, dec2oct</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
Returns:&nbsp;&nbsp;&nbsp;The decimal argument converted to hexadecimal, binary or octal<br>
</span>
<hr style="width: 100%; height: 2px;"><span style="font-family: monospace;"></span>
<h2>Associative Arrays</h2>
An associate array maps a key value (a string with no spaces in it) to
a single value (any string).&nbsp;&nbsp;&nbsp; <br>
<b><br>
</b>
<hr style="width: 100%; height: 2px;"><b>set</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of associative
array<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: The key
value to associate<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 3: The
value associated with the key<br>
Returns:&nbsp;&nbsp;&nbsp;Nothing<br>
</span>
<hr><b>get</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of associative
array<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: The key
to retrieve<br>
Returns:&nbsp;&nbsp;&nbsp;The value stored in the array for that key<br>
</span>
<hr><b>keys</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of associative
array<br>
Returns:&nbsp;&nbsp;&nbsp;Returns a list of all defined keys in the
array<br>
</span>
<hr><b>defined</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of associative
array<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: The key
to test<br>
Returns:&nbsp;&nbsp;&nbsp;Returns true if the key is defined (i.e. not
empty)<br>
</span>
<hr style="width: 100%; height: 2px;"><span style="font-family: monospace;"></span>
<h2>Named Stacks</h2>
A stack is an ordered list of strings (with no spaces in them).<br>
<br>
<hr style="width: 100%; height: 2px;"><b>push</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of stack<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Value
to push onto the top of the stack (must not contain<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; a space)<br>
Returns:&nbsp;&nbsp;&nbsp;None<br>
</span>
<hr><b>pop</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of stack<br>
Returns:&nbsp;&nbsp;&nbsp;Top element from the stack after removing it<br>
</span>
<hr><b>peek</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of stack<br>
Returns:&nbsp;&nbsp;&nbsp;Top element from the stack without removing it<br>
</span>
<hr><b>depth</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of stack<br>
Returns:&nbsp;&nbsp;&nbsp;Number of items on the stack<br>
</span>
<hr style="width: 100%; height: 2px;"><span style="font-family: monospace;"></span>
<h2>Function memoization</h2>
To reduce the number of calls to slow functions (such as $(shell) a single memoization function is provided.<br>
<br>
<hr style="width: 100%; height: 2px;"><b>memoize</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of function to memoize<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: String argument for the function<br>
Returns:&nbsp;&nbsp;&nbsp;Result of $1 applied to $2 but only calls $1 once for each unique $2<br>
</span>
<hr style="width: 100%; height: 2px;"><span style="font-family: monospace;"></span>
<h2>Miscellaneous and Debugging Facilities</h2>
GMSL defines the following constants; all are accessed as normal GNU
Make variables by wrapping them in <span style="font-family: monospace;">$()</span> or <span style="font-family: monospace;">${}</span>.<br>
<br>
<table style="text-align: left;" border="1" cellpadding="2" cellspacing="2">
<tbody>
<tr>
<td><span style="font-style: italic;">Constant</span><br>
</td>
<td><span style="font-style: italic;">Value</span><br>
</td>
<td><span style="font-style: italic;">Purpose</span><br>
</td>
</tr>
<tr>
<td><span style="font-family: monospace;">true</span><br>
</td>
<td><span style="font-family: monospace;">T</span><br>
</td>
<td>Boolean for <span style="font-family: monospace;">$(if)</span>
and return from&nbsp; GMSL functions<br>
</td>
</tr>
<tr>
<td><span style="font-family: monospace;">false</span><br>
</td>
<td><br>
</td>
<td>Boolean for <span style="font-family: monospace;">$(if)</span>
and return from GMSL functions<br>
</td>
</tr>
<tr>
<td><span style="font-family: monospace;">gmsl_version</span><br>
</td>
<td><span style="font-family: monospace;">1 0 0</span><br>
</td>
<td>GMSL version number as list: major minor revision<br>
</td>
</tr>
</tbody>
</table>
<span style="font-weight: bold;"><br>
gmsl_compatible</span><span style="font-family: monospace;"><br>
<br>
Arguments: List containing the desired library version number (maj min
rev)<br>
</span><span style="font-family: monospace;">Returns:&nbsp;&nbsp;
$(true) if this version of the library is compatible<br>
</span><span style="font-family: monospace;">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
with the requested version number, otherwise $(false)</span>
<hr><b>gmsl-print-% (target not a function)</b><br>
<br>
<span style="font-family: monospace;">Arguments: The % should be
replaced by the name of a variable that you<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; wish to
print out.<br>
Action:&nbsp;&nbsp;&nbsp; Echos the name of the variable that matches
the % and its value.<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; For
example, 'make gmsl-print-SHELL' will output the value of<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; the SHELL
variable<br>
</span>
<hr><b>gmsl-echo-% (target not a function)</b><br>
<br>
<span style="font-family: monospace;">Arguments: The % should be
replaced by the name of a variable that you<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; wish to
print out.<br>
Action:&nbsp;&nbsp;&nbsp; Echos the value of the variable that matches
the %.<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; For
example, 'make gmsl-echo-SHELL' will output the value of<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; the SHELL
variable<br>
</span>
<hr><b>assert</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A boolean that must
be true or the assertion will fail<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: The
message to print with the assertion<br>
Returns:&nbsp;&nbsp;&nbsp;None<br>
</span>
<hr><b>assert_exists</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of file that
must exist, if it is missing an assertion<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; will be
generated<br>
Returns:&nbsp;&nbsp;&nbsp;None<br>
</span>
<hr style="width: 100%; height: 2px;"><br>
GMSL has a number of environment variables (or command-line overrides)
that control various bits of functionality:<br>
<br>
<table style="text-align: left;" border="1" cellpadding="2" cellspacing="2">
<tbody>
<tr>
<td><span style="font-style: italic;">Variable</span><br>
</td>
<td><span style="font-style: italic;">Purpose</span><br>
</td>
</tr>
<tr>
<td><span style="font-family: monospace;">GMSL_NO_WARNINGS</span><br>
</td>
<td>If set prevents GMSL from outputting warning messages:
artithmetic functions generate underflow warnings.<br>
</td>
</tr>
<tr>
<td><span style="font-family: monospace;">GMSL_NO_ERRORS</span><br>
</td>
<td>If set prevents GMSL from generating fatal errors: division
by zero or failed assertions are fatal.<br>
</td>
</tr>
<tr>
<td><span style="font-family: monospace;">GMSL_TRACE</span><br>
</td>
<td>Enables function tracing.&nbsp; Calls to GMSL functions will
result in name and arguments being traced.<br>
</td>
</tr>
</tbody>
</table>
<span style="font-family: monospace;"></span><br>
<hr>
Copyright (c) 2005-2022 <a href="https://www.jgc.org/">John Graham-Cumming</a>.<br>
<hr style="width: 100%; height: 2px;">
</body></html>

8
mk/platform/unix_common.mk
View file

@ -7,14 +7,8 @@ MKDIR=mkdir -p
# Go should not be required to run functional tests
GOOS ?= $(shell go env GOOS)
# Current versioning information from env
#Current versioning information from env
BUILD_VERSION?=$(shell git describe --tags)
BUILD_TIMESTAMP=$(shell date +%F"_"%T)
DEFAULT_CONFIGDIR?=/etc/crowdsec
DEFAULT_DATADIR?=/var/lib/crowdsec/data
PKG_CONFIG:=$(shell command -v pkg-config 2>/dev/null)
# See if we have libre2-dev installed for C++ optimizations.
# In fedora and other distros, we need to tell where to find re2.pc
RE2_CHECK := $(shell PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$(PKG_CONFIG_PATH) pkg-config --libs re2 2>/dev/null)

8
mk/platform/windows.mk
View file

@ -4,9 +4,9 @@ MAKE=make
GOOS=windows
PREFIX=$(shell $$env:TEMP)
# Current versioning information from env
# BUILD_VERSION?=$(shell (Invoke-WebRequest -UseBasicParsing -Uri https://api.github.com/repos/crowdsecurity/crowdsec/releases/latest).Content | jq -r '.tag_name')
# hardcode it till I find a workaround
#Current versioning information from env
#BUILD_VERSION?=$(shell (Invoke-WebRequest -UseBasicParsing -Uri https://api.github.com/repos/crowdsecurity/crowdsec/releases/latest).Content | jq -r '.tag_name')
#hardcode it till i find a workaround
BUILD_VERSION?=$(shell git describe --tags $$(git rev-list --tags --max-count=1))
BUILD_TIMESTAMP?=$(shell Get-Date -Format "yyyy-MM-dd_HH:mm:ss")
DEFAULT_CONFIGDIR?=C:\\ProgramData\\CrowdSec\\config
@ -18,5 +18,3 @@ CP=Copy-Item
CPR=Copy-Item -Recurse
MKDIR=New-Item -ItemType directory
WIN_IGNORE_ERR=; exit 0
PKG_CONFIG:=$(shell Get-Command pkg-config -ErrorAction SilentlyContinue)

40
pkg/acquisition/acquisition.go
View file

@ -35,20 +35,6 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/types"
)
type DataSourceUnavailableError struct {
Name string
Err error
}
func (e *DataSourceUnavailableError) Error() string {
return fmt.Sprintf("datasource '%s' is not available: %v", e.Name, e.Err)
}
func (e *DataSourceUnavailableError) Unwrap() error {
return e.Err
}
// The interface each datasource must implement
type DataSource interface {
GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module
@ -89,10 +75,6 @@ func GetDataSourceIface(dataSourceType string) DataSource {
return source()
}
// DataSourceConfigure creates and returns a DataSource object from a configuration,
// if the configuration is not valid it returns an error.
// If the datasource can't be run (eg. journalctl not available), it still returns an error which
// can be checked for the appropriate action.
func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataSource, error) {
// we dump it back to []byte, because we want to decode the yaml blob twice:
// once to DataSourceCommonCfg, and then later to the dedicated type of the datasource
@ -118,7 +100,7 @@ func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataS
subLogger := clog.WithFields(customLog)
/* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */
if err := dataSrc.CanRun(); err != nil {
return nil, &DataSourceUnavailableError{Name: commonConfig.Source, Err: err}
return nil, fmt.Errorf("datasource %s cannot be run: %w", commonConfig.Source, err)
}
/* configure the actual datasource */
if err := dataSrc.Configure(yamlConfig, subLogger); err != nil {
@ -191,11 +173,10 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
}
dec := yaml.NewDecoder(yamlFile)
dec.SetStrict(true)
idx := -1
for {
var sub configuration.DataSourceCommonCfg
var idx int
err = dec.Decode(&sub)
idx += 1
if err != nil {
if !errors.Is(err, io.EOF) {
return nil, fmt.Errorf("failed to yaml decode %s: %w", acquisFile, err)
@ -212,6 +193,7 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
if len(sub.Labels) == 0 {
if sub.Source == "" {
log.Debugf("skipping empty item in %s", acquisFile)
idx += 1
continue
}
return nil, fmt.Errorf("missing labels in %s (position: %d)", acquisFile, idx)
@ -226,11 +208,6 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
sub.UniqueId = uniqueId
src, err := DataSourceConfigure(sub)
if err != nil {
var dserr *DataSourceUnavailableError
if errors.As(err, &dserr) {
log.Error(err)
continue
}
return nil, fmt.Errorf("while configuring datasource of type %s from %s (position: %d): %w", sub.Source, acquisFile, idx, err)
}
if sub.TransformExpr != "" {
@ -241,6 +218,7 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
transformRuntimes[uniqueId] = vm
}
sources = append(sources, *src)
idx += 1
}
}
return sources, nil
@ -317,11 +295,6 @@ func transform(transformChan chan types.Event, output chan types.Event, AcquisTo
}
func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb *tomb.Tomb) error {
// Don't wait if we have no sources, as it will hang forever
if len(sources) == 0 {
return nil
}
for i := 0; i < len(sources); i++ {
subsrc := sources[i] //ensure its a copy
log.Debugf("starting one source %d/%d ->> %T", i, len(sources), subsrc)
@ -357,8 +330,11 @@ func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb
return nil
})
}
// Don't wait if we have no sources, as it will hang forever
if len(sources) > 0 {
/*return only when acquisition is over (cat) or never (tail)*/
err := AcquisTomb.Wait()
return err
}
return nil
}

2
pkg/acquisition/acquisition_test.go
View file

@ -167,7 +167,7 @@ log_level: debug
source: mock_cant_run
wowo: ajsajasjas
`,
ExpectedError: "datasource 'mock_cant_run' is not available: can't run bro",
ExpectedError: "datasource mock_cant_run cannot be run: can't run bro",
},
}

16
pkg/acquisition/modules/cloudwatch/cloudwatch.go
View file

@ -13,6 +13,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
@ -200,7 +201,7 @@ func (cw *CloudwatchSource) Configure(yamlConfig []byte, logger *log.Entry) erro
targetStream := "*"
if cw.Config.StreamRegexp != nil {
if _, err := regexp.Compile(*cw.Config.StreamRegexp); err != nil {
return fmt.Errorf("while compiling regexp '%s': %w", *cw.Config.StreamRegexp, err)
return errors.Wrapf(err, "error while compiling regexp '%s'", *cw.Config.StreamRegexp)
}
targetStream = *cw.Config.StreamRegexp
} else if cw.Config.StreamName != nil {
@ -344,7 +345,8 @@ func (cw *CloudwatchSource) WatchLogGroupForStreams(out chan LogStreamTailConfig
},
)
if err != nil {
return fmt.Errorf("while describing group %s: %w", cw.Config.GroupName, err)
newerr := errors.Wrapf(err, "while describing group %s", cw.Config.GroupName)
return newerr
}
cw.logger.Tracef("after DescribeLogStreamsPagesWithContext")
}
@ -493,7 +495,7 @@ func (cw *CloudwatchSource) TailLogStream(cfg *LogStreamTailConfig, outChan chan
},
)
if err != nil {
newerr := fmt.Errorf("while reading %s/%s: %w", cfg.GroupName, cfg.StreamName, err)
newerr := errors.Wrapf(err, "while reading %s/%s", cfg.GroupName, cfg.StreamName)
cfg.logger.Warningf("err : %s", newerr)
return newerr
}
@ -530,7 +532,7 @@ func (cw *CloudwatchSource) ConfigureByDSN(dsn string, labels map[string]string,
u, err := url.ParseQuery(args[1])
if err != nil {
return fmt.Errorf("while parsing %s: %w", dsn, err)
return errors.Wrapf(err, "while parsing %s", dsn)
}
for k, v := range u {
@ -541,7 +543,7 @@ func (cw *CloudwatchSource) ConfigureByDSN(dsn string, labels map[string]string,
}
lvl, err := log.ParseLevel(v[0])
if err != nil {
return fmt.Errorf("unknown level %s: %w", v[0], err)
return errors.Wrapf(err, "unknown level %s", v[0])
}
cw.logger.Logger.SetLevel(lvl)
@ -575,7 +577,7 @@ func (cw *CloudwatchSource) ConfigureByDSN(dsn string, labels map[string]string,
//let's reuse our parser helper so that a ton of date formats are supported
duration, err := time.ParseDuration(v[0])
if err != nil {
return fmt.Errorf("unable to parse '%s' as duration: %w", v[0], err)
return errors.Wrapf(err, "unable to parse '%s' as duration", v[0])
}
cw.logger.Debugf("parsed '%s' as '%s'", v[0], duration)
start := time.Now().UTC().Add(-duration)
@ -672,7 +674,7 @@ func (cw *CloudwatchSource) CatLogStream(cfg *LogStreamTailConfig, outChan chan
},
)
if err != nil {
return fmt.Errorf("while reading logs from %s/%s: %w", cfg.GroupName, cfg.StreamName, err)
return errors.Wrapf(err, "while reading logs from %s/%s", cfg.GroupName, cfg.StreamName)
}
cfg.logger.Tracef("after GetLogEventsPagesWithContext")
case <-cw.t.Dying():

7
pkg/acquisition/modules/docker/docker.go
View file

@ -12,6 +12,7 @@ import (
dockerTypes "github.com/docker/docker/api/types"
"github.com/docker/docker/client"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
@ -79,7 +80,7 @@ func (d *DockerSource) UnmarshalConfig(yamlConfig []byte) error {
err := yaml.UnmarshalStrict(yamlConfig, &d.Config)
if err != nil {
return fmt.Errorf("while parsing DockerAcquisition configuration: %w", err)
return errors.Wrap(err, "Cannot parse DockerAcquisition configuration")
}
if d.logger != nil {
@ -213,7 +214,7 @@ func (d *DockerSource) ConfigureByDSN(dsn string, labels map[string]string, logg
parameters, err := url.ParseQuery(args[1])
if err != nil {
return fmt.Errorf("while parsing parameters %s: %w", dsn, err)
return errors.Wrapf(err, "while parsing parameters %s: %s", dsn, err)
}
for k, v := range parameters {
@ -224,7 +225,7 @@ func (d *DockerSource) ConfigureByDSN(dsn string, labels map[string]string, logg
}
lvl, err := log.ParseLevel(v[0])
if err != nil {
return fmt.Errorf("unknown level %s: %w", v[0], err)
return errors.Wrapf(err, "unknown level %s", v[0])
}
d.logger.Logger.SetLevel(lvl)
case "until":

22
pkg/acquisition/modules/file/file.go
View file

@ -14,6 +14,8 @@ import (
"strings"
"time"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/fsnotify/fsnotify"
"github.com/nxadm/tail"
"github.com/pkg/errors"
@ -22,8 +24,6 @@ import (
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
@ -110,7 +110,7 @@ func (f *FileSource) Configure(yamlConfig []byte, logger *log.Entry) error {
f.watcher, err = fsnotify.NewWatcher()
if err != nil {
return fmt.Errorf("could not create fsnotify watcher: %w", err)
return errors.Wrapf(err, "Could not create fsnotify watcher")
}
f.logger.Tracef("Actual FileAcquisition Configuration %+v", f.config)
@ -130,7 +130,7 @@ func (f *FileSource) Configure(yamlConfig []byte, logger *log.Entry) error {
}
files, err := filepath.Glob(pattern)
if err != nil {
return fmt.Errorf("glob failure: %w", err)
return errors.Wrap(err, "Glob failure")
}
if len(files) == 0 {
f.logger.Warnf("No matching files for pattern %s", pattern)
@ -191,7 +191,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger
if len(args) == 2 && len(args[1]) != 0 {
params, err := url.ParseQuery(args[1])
if err != nil {
return fmt.Errorf("could not parse file args: %w", err)
return errors.Wrap(err, "could not parse file args")
}
for key, value := range params {
switch key {
@ -201,7 +201,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger
}
lvl, err := log.ParseLevel(value[0])
if err != nil {
return fmt.Errorf("unknown level %s: %w", value[0], err)
return errors.Wrapf(err, "unknown level %s", value[0])
}
f.logger.Logger.SetLevel(lvl)
case "max_buffer_size":
@ -210,7 +210,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger
}
maxBufferSize, err := strconv.Atoi(value[0])
if err != nil {
return fmt.Errorf("could not parse max_buffer_size %s: %w", value[0], err)
return errors.Wrapf(err, "could not parse max_buffer_size %s", value[0])
}
f.config.MaxBufferSize = maxBufferSize
default:
@ -226,7 +226,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger
f.logger.Debugf("Will try pattern %s", args[0])
files, err := filepath.Glob(args[0])
if err != nil {
return fmt.Errorf("glob failure: %w", err)
return errors.Wrap(err, "Glob failure")
}
if len(files) == 0 {
@ -433,7 +433,7 @@ func (f *FileSource) monitorNewFiles(out chan types.Event, t *tomb.Tomb) error {
case <-t.Dying():
err := f.watcher.Close()
if err != nil {
return fmt.Errorf("could not remove all inotify watches: %w", err)
return errors.Wrapf(err, "could not remove all inotify watches")
}
return nil
}
@ -495,7 +495,7 @@ func (f *FileSource) readFile(filename string, out chan types.Event, t *tomb.Tom
fd, err := os.Open(filename)
if err != nil {
return fmt.Errorf("failed opening %s: %w", filename, err)
return errors.Wrapf(err, "failed opening %s", filename)
}
defer fd.Close()
@ -503,7 +503,7 @@ func (f *FileSource) readFile(filename string, out chan types.Event, t *tomb.Tom
gz, err := gzip.NewReader(fd)
if err != nil {
logger.Errorf("Failed to read gz file: %s", err)
return fmt.Errorf("failed to read gz %s: %w", filename, err)
return errors.Wrapf(err, "failed to read gz %s", filename)
}
defer gz.Close()
scanner = bufio.NewScanner(gz)

4
pkg/acquisition/modules/file/file_test.go
View file

@ -38,7 +38,7 @@ func TestBadConfiguration(t *testing.T) {
{
name: "glob syntax error",
config: `filename: "[asd-.log"`,
expectedErr: "glob failure: syntax error in pattern",
expectedErr: "Glob failure: syntax error in pattern",
},
{
name: "bad exclude regexp",
@ -150,7 +150,7 @@ filename: /`,
config: `
mode: cat
filename: "[*-.log"`,
expectedConfigErr: "glob failure: syntax error in pattern",
expectedConfigErr: "Glob failure: syntax error in pattern",
logLevel: log.WarnLevel,
expectedLines: 0,
},

3
pkg/acquisition/modules/journalctl/journalctl.go
View file

@ -9,6 +9,7 @@ import (
"strings"
"time"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
@ -236,7 +237,7 @@ func (j *JournalCtlSource) ConfigureByDSN(dsn string, labels map[string]string,
}
lvl, err := log.ParseLevel(value[0])
if err != nil {
return fmt.Errorf("unknown level %s: %w", value[0], err)
return errors.Wrapf(err, "unknown level %s", value[0])
}
j.logger.Logger.SetLevel(lvl)
case "since":

11
pkg/acquisition/modules/kafka/kafka.go
View file

@ -10,6 +10,7 @@ import (
"strconv"
"time"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/segmentio/kafka-go"
log "github.com/sirupsen/logrus"
@ -92,12 +93,12 @@ func (k *KafkaSource) Configure(yamlConfig []byte, logger *log.Entry) error {
dialer, err := k.Config.NewDialer()
if err != nil {
return fmt.Errorf("cannot create %s dialer: %w", dataSourceName, err)
return errors.Wrapf(err, "cannot create %s dialer", dataSourceName)
}
k.Reader, err = k.Config.NewReader(dialer)
if err != nil {
return fmt.Errorf("cannote create %s reader: %w", dataSourceName, err)
return errors.Wrapf(err, "cannote create %s reader", dataSourceName)
}
if k.Reader == nil {
@ -148,7 +149,7 @@ func (k *KafkaSource) ReadMessage(out chan types.Event) error {
if err == io.EOF {
return nil
}
k.logger.Errorln(fmt.Errorf("while reading %s message: %w", dataSourceName, err))
k.logger.Errorln(errors.Wrapf(err, "while reading %s message", dataSourceName))
}
l := types.Line{
Raw: string(m.Value),
@ -180,7 +181,7 @@ func (k *KafkaSource) RunReader(out chan types.Event, t *tomb.Tomb) error {
case <-t.Dying():
k.logger.Infof("%s datasource topic %s stopping", dataSourceName, k.Config.Topic)
if err := k.Reader.Close(); err != nil {
return fmt.Errorf("while closing %s reader on topic '%s': %w", dataSourceName, k.Config.Topic, err)
return errors.Wrapf(err, "while closing %s reader on topic '%s'", dataSourceName, k.Config.Topic)
}
return nil
}
@ -263,7 +264,7 @@ func (kc *KafkaConfiguration) NewReader(dialer *kafka.Dialer) (*kafka.Reader, er
rConf.GroupID = kc.GroupID
}
if err := rConf.Validate(); err != nil {
return &kafka.Reader{}, fmt.Errorf("while validating reader configuration: %w", err)
return &kafka.Reader{}, errors.Wrapf(err, "while validating reader configuration")
}
return kafka.NewReader(rConf), nil
}

13
pkg/acquisition/modules/kafka/kafka_test.go
View file

@ -8,14 +8,13 @@ import (
"testing"
"time"
"github.com/segmentio/kafka-go"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
"gopkg.in/tomb.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/cstest"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/segmentio/kafka-go"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
"gotest.tools/v3/assert"
)
func TestConfigure(t *testing.T) {
@ -179,7 +178,7 @@ topic: crowdsecplaintext`), subLogger)
break READLOOP
}
}
require.Equal(t, ts.expectedLines, actualLines)
assert.Equal(t, ts.expectedLines, actualLines)
tomb.Kill(nil)
tomb.Wait()
})
@ -255,7 +254,7 @@ tls:
break READLOOP
}
}
require.Equal(t, ts.expectedLines, actualLines)
assert.Equal(t, ts.expectedLines, actualLines)
tomb.Kill(nil)
tomb.Wait()
})

37
pkg/acquisition/modules/kinesis/kinesis.go
View file

@ -13,6 +13,7 @@ import (
"github.com/aws/aws-sdk-go/aws/arn"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/kinesis"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
@ -123,7 +124,7 @@ func (k *KinesisSource) UnmarshalConfig(yamlConfig []byte) error {
err := yaml.UnmarshalStrict(yamlConfig, &k.Config)
if err != nil {
return fmt.Errorf("Cannot parse kinesis datasource configuration: %w", err)
return errors.Wrap(err, "Cannot parse kinesis datasource configuration")
}
if k.Config.Mode == "" {
@ -217,7 +218,7 @@ func (k *KinesisSource) WaitForConsumerDeregistration(consumerName string, strea
return nil
default:
k.logger.Errorf("Error while waiting for consumer deregistration: %s", err)
return fmt.Errorf("cannot describe stream consumer: %w", err)
return errors.Wrap(err, "Cannot describe stream consumer")
}
}
time.Sleep(time.Millisecond * 200 * time.Duration(i+1))
@ -235,12 +236,12 @@ func (k *KinesisSource) DeregisterConsumer() error {
switch err.(type) {
case *kinesis.ResourceNotFoundException:
default:
return fmt.Errorf("cannot deregister stream consumer: %w", err)
return errors.Wrap(err, "Cannot deregister stream consumer")
}
}
err = k.WaitForConsumerDeregistration(k.Config.ConsumerName, k.Config.StreamARN)
if err != nil {
return fmt.Errorf("cannot wait for consumer deregistration: %w", err)
return errors.Wrap(err, "Cannot wait for consumer deregistration")
}
return nil
}
@ -252,7 +253,7 @@ func (k *KinesisSource) WaitForConsumerRegistration(consumerARN string) error {
ConsumerARN: aws.String(consumerARN),
})
if err != nil {
return fmt.Errorf("cannot describe stream consumer: %w", err)
return errors.Wrap(err, "Cannot describe stream consumer")
}
if *describeOutput.ConsumerDescription.ConsumerStatus == "ACTIVE" {
k.logger.Debugf("Consumer %s is active", consumerARN)
@ -271,11 +272,11 @@ func (k *KinesisSource) RegisterConsumer() (*kinesis.RegisterStreamConsumerOutpu
StreamARN: aws.String(k.Config.StreamARN),
})
if err != nil {
return nil, fmt.Errorf("cannot register stream consumer: %w", err)
return nil, errors.Wrap(err, "Cannot register stream consumer")
}
err = k.WaitForConsumerRegistration(*streamConsumer.Consumer.ConsumerARN)
if err != nil {
return nil, fmt.Errorf("timeout while waiting for consumer to be active: %w", err)
return nil, errors.Wrap(err, "Timeout while waiting for consumer to be active")
}
return streamConsumer, nil
}
@ -338,7 +339,7 @@ func (k *KinesisSource) ReadFromSubscription(reader kinesis.SubscribeToShardEven
logger.Infof("Subscribed shard reader is dying")
err := reader.Close()
if err != nil {
return fmt.Errorf("cannot close kinesis subscribed shard reader: %w", err)
return errors.Wrap(err, "Cannot close kinesis subscribed shard reader")
}
return nil
case event, ok := <-reader.Events():
@ -361,7 +362,7 @@ func (k *KinesisSource) SubscribeToShards(arn arn.ARN, streamConsumer *kinesis.R
StreamName: aws.String(arn.Resource[7:]),
})
if err != nil {
return fmt.Errorf("cannot list shards for enhanced_read: %w", err)
return errors.Wrap(err, "Cannot list shards for enhanced_read")
}
for _, shard := range shards.Shards {
@ -372,7 +373,7 @@ func (k *KinesisSource) SubscribeToShards(arn arn.ARN, streamConsumer *kinesis.R
ConsumerARN: streamConsumer.Consumer.ConsumerARN,
})
if err != nil {
return fmt.Errorf("cannot subscribe to shard: %w", err)
return errors.Wrap(err, "Cannot subscribe to shard")
}
k.shardReaderTomb.Go(func() error {
return k.ReadFromSubscription(r.GetEventStream().Reader, out, shardId, arn.Resource[7:])
@ -384,7 +385,7 @@ func (k *KinesisSource) SubscribeToShards(arn arn.ARN, streamConsumer *kinesis.R
func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error {
parsedARN, err := arn.Parse(k.Config.StreamARN)
if err != nil {
return fmt.Errorf("cannot parse stream ARN: %w", err)
return errors.Wrap(err, "Cannot parse stream ARN")
}
if !strings.HasPrefix(parsedARN.Resource, "stream/") {
return fmt.Errorf("resource part of stream ARN %s does not start with stream/", k.Config.StreamARN)
@ -394,12 +395,12 @@ func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error {
k.logger.Info("starting kinesis acquisition with enhanced fan-out")
err = k.DeregisterConsumer()
if err != nil {
return fmt.Errorf("cannot deregister consumer: %w", err)
return errors.Wrap(err, "Cannot deregister consumer")
}
streamConsumer, err := k.RegisterConsumer()
if err != nil {
return fmt.Errorf("cannot register consumer: %w", err)
return errors.Wrap(err, "Cannot register consumer")
}
for {
@ -407,7 +408,7 @@ func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error {
err = k.SubscribeToShards(parsedARN, streamConsumer, out)
if err != nil {
return fmt.Errorf("cannot subscribe to shards: %w", err)
return errors.Wrap(err, "Cannot subscribe to shards")
}
select {
case <-t.Dying():
@ -416,7 +417,7 @@ func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error {
_ = k.shardReaderTomb.Wait() //we don't care about the error as we kill the tomb ourselves
err = k.DeregisterConsumer()
if err != nil {
return fmt.Errorf("cannot deregister consumer: %w", err)
return errors.Wrap(err, "Cannot deregister consumer")
}
return nil
case <-k.shardReaderTomb.Dying():
@ -439,7 +440,7 @@ func (k *KinesisSource) ReadFromShard(out chan types.Event, shardId string) erro
ShardIteratorType: aws.String(kinesis.ShardIteratorTypeLatest)})
if err != nil {
logger.Errorf("Cannot get shard iterator: %s", err)
return fmt.Errorf("cannot get shard iterator: %w", err)
return errors.Wrap(err, "Cannot get shard iterator")
}
it := sharIt.ShardIterator
//AWS recommends to wait for a second between calls to GetRecords for a given shard
@ -460,7 +461,7 @@ func (k *KinesisSource) ReadFromShard(out chan types.Event, shardId string) erro
continue
default:
logger.Error("Cannot get records")
return fmt.Errorf("cannot get records: %w", err)
return errors.Wrap(err, "Cannot get records")
}
}
k.ParseAndPushRecords(records.Records, out, logger, shardId)
@ -485,7 +486,7 @@ func (k *KinesisSource) ReadFromStream(out chan types.Event, t *tomb.Tomb) error
StreamName: aws.String(k.Config.StreamName),
})
if err != nil {
return fmt.Errorf("cannot list shards: %w", err)
return errors.Wrap(err, "Cannot list shards")
}
k.shardReaderTomb = &tomb.Tomb{}
for _, shard := range shards.Shards {

14
pkg/acquisition/modules/kubernetesaudit/k8s_audit.go
View file

@ -8,16 +8,16 @@ import (
"net/http"
"strings"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
"k8s.io/apiserver/pkg/apis/audit"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
type KubernetesAuditConfiguration struct {
@ -66,7 +66,7 @@ func (ka *KubernetesAuditSource) UnmarshalConfig(yamlConfig []byte) error {
k8sConfig := KubernetesAuditConfiguration{}
err := yaml.UnmarshalStrict(yamlConfig, &k8sConfig)
if err != nil {
return fmt.Errorf("cannot parse k8s-audit configuration: %w", err)
return errors.Wrap(err, "Cannot parse k8s-audit configuration")
}
ka.config = k8sConfig
@ -140,7 +140,7 @@ func (ka *KubernetesAuditSource) StreamingAcquisition(out chan types.Event, t *t
t.Go(func() error {
err := ka.server.ListenAndServe()
if err != nil && err != http.ErrServerClosed {
return fmt.Errorf("k8s-audit server failed: %w", err)
return errors.Wrap(err, "k8s-audit server failed")
}
return nil
})

13
pkg/acquisition/modules/s3/s3.go
View file

@ -6,7 +6,6 @@ import (
"compress/gzip"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/url"
@ -22,13 +21,13 @@ import (
"github.com/aws/aws-sdk-go/service/s3/s3iface"
"github.com/aws/aws-sdk-go/service/sqs"
"github.com/aws/aws-sdk-go/service/sqs/sqsiface"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
type S3Configuration struct {
@ -564,7 +563,7 @@ func (s *S3Source) ConfigureByDSN(dsn string, labels map[string]string, logger *
if len(args) == 2 && len(args[1]) != 0 {
params, err := url.ParseQuery(args[1])
if err != nil {
return fmt.Errorf("could not parse s3 args: %w", err)
return errors.Wrap(err, "could not parse s3 args")
}
for key, value := range params {
switch key {
@ -574,7 +573,7 @@ func (s *S3Source) ConfigureByDSN(dsn string, labels map[string]string, logger *
}
lvl, err := log.ParseLevel(value[0])
if err != nil {
return fmt.Errorf("unknown level %s: %w", value[0], err)
return errors.Wrapf(err, "unknown level %s", value[0])
}
s.logger.Logger.SetLevel(lvl)
case "max_buffer_size":
@ -583,7 +582,7 @@ func (s *S3Source) ConfigureByDSN(dsn string, labels map[string]string, logger *
}
maxBufferSize, err := strconv.Atoi(value[0])
if err != nil {
return fmt.Errorf("invalid value for 'max_buffer_size': %w", err)
return errors.Wrapf(err, "invalid value for 'max_buffer_size'")
}
s.logger.Debugf("Setting max buffer size to %d", maxBufferSize)
s.Config.MaxBufferSize = maxBufferSize

9
pkg/acquisition/modules/syslog/internal/server/syslogserver.go
View file

@ -6,6 +6,7 @@ import (
"strings"
"time"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
)
@ -30,18 +31,18 @@ func (s *SyslogServer) Listen(listenAddr string, port int) error {
s.port = port
udpAddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", s.listenAddr, s.port))
if err != nil {
return fmt.Errorf("could not resolve addr %s: %w", s.listenAddr, err)
return errors.Wrapf(err, "could not resolve addr %s", s.listenAddr)
}
udpConn, err := net.ListenUDP("udp", udpAddr)
if err != nil {
return fmt.Errorf("could not listen on port %d: %w", s.port, err)
return errors.Wrapf(err, "could not listen on port %d", s.port)
}
s.Logger.Debugf("listening on %s:%d", s.listenAddr, s.port)
s.udpConn = udpConn
err = s.udpConn.SetReadDeadline(time.Now().UTC().Add(100 * time.Millisecond))
if err != nil {
return fmt.Errorf("could not set read deadline on UDP socket: %w", err)
return errors.Wrap(err, "could not set read deadline on UDP socket")
}
return nil
}
@ -86,7 +87,7 @@ func (s *SyslogServer) StartServer() *tomb.Tomb {
func (s *SyslogServer) KillServer() error {
err := s.udpConn.Close()
if err != nil {
return fmt.Errorf("could not close UDP connection: %w", err)
return errors.Wrap(err, "could not close UDP connection")
}
close(s.channel)
return nil

5
pkg/acquisition/modules/syslog/syslog.go
View file

@ -6,6 +6,7 @@ import (
"strings"
"time"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
@ -99,7 +100,7 @@ func (s *SyslogSource) UnmarshalConfig(yamlConfig []byte) error {
err := yaml.UnmarshalStrict(yamlConfig, &s.config)
if err != nil {
return fmt.Errorf("cannot parse syslog configuration: %w", err)
return errors.Wrap(err, "Cannot parse syslog configuration")
}
if s.config.Addr == "" {
@ -139,7 +140,7 @@ func (s *SyslogSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb)
s.server.SetChannel(c)
err := s.server.Listen(s.config.Addr, s.config.Port)
if err != nil {
return fmt.Errorf("could not start syslog server: %w", err)
return errors.Wrap(err, "could not start syslog server")
}
s.serverTomb = s.server.StartServer()
t.Go(func() error {

10
pkg/apiclient/alerts_service.go
View file

@ -5,9 +5,9 @@ import (
"fmt"
"net/http"
qs "github.com/google/go-querystring/query"
"github.com/crowdsecurity/crowdsec/pkg/models"
qs "github.com/google/go-querystring/query"
"github.com/pkg/errors"
)
// type ApiAlerts service
@ -72,7 +72,7 @@ func (s *AlertsService) List(ctx context.Context, opts AlertsListOpts) (*models.
u := fmt.Sprintf("%s/alerts", s.client.URLPrefix)
params, err := qs.Values(opts)
if err != nil {
return nil, nil, fmt.Errorf("building query: %w", err)
return nil, nil, errors.Wrap(err, "building query")
}
if len(params) > 0 {
URI = fmt.Sprintf("%s?%s", u, params.Encode())
@ -82,12 +82,12 @@ func (s *AlertsService) List(ctx context.Context, opts AlertsListOpts) (*models.
req, err := s.client.NewRequest(http.MethodGet, URI, nil)
if err != nil {
return nil, nil, fmt.Errorf("building request: %w", err)
return nil, nil, errors.Wrap(err, "building request")
}
resp, err := s.client.Do(ctx, req, &alerts)
if err != nil {
return nil, resp, fmt.Errorf("performing request: %w", err)
return nil, resp, errors.Wrap(err, "performing request")
}
return &alerts, resp, nil
}

7
pkg/apiclient/alerts_service_test.go
View file

@ -8,13 +8,12 @@ import (
"reflect"
"testing"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/models"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestAlertsListAsMachine(t *testing.T) {

30
pkg/apiclient/auth.go
View file

@ -3,21 +3,23 @@ package apiclient
import (
"bytes"
"encoding/json"
"fmt"
"io"
"math/rand"
"net/http"
"net/http/httputil"
"net/url"
"sync"
"time"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
//"errors"
"fmt"
"io"
"net/http"
"net/http/httputil"
"net/url"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
//"google.golang.org/appengine/log"
)
type APIKeyTransport struct {
@ -167,11 +169,11 @@ func (t *JWTTransport) refreshJwtToken() error {
enc.SetEscapeHTML(false)
err = enc.Encode(auth)
if err != nil {
return fmt.Errorf("could not encode jwt auth body: %w", err)
return errors.Wrap(err, "could not encode jwt auth body")
}
req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s%s/watchers/login", t.URL, t.VersionPrefix), buf)
if err != nil {
return fmt.Errorf("could not create request: %w", err)
return errors.Wrap(err, "could not create request")
}
req.Header.Add("Content-Type", "application/json")
client := &http.Client{
@ -194,7 +196,7 @@ func (t *JWTTransport) refreshJwtToken() error {
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("could not get jwt token: %w", err)
return errors.Wrap(err, "could not get jwt token")
}
log.Debugf("auth-jwt : http %d", resp.StatusCode)
@ -215,10 +217,10 @@ func (t *JWTTransport) refreshJwtToken() error {
}
if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
return fmt.Errorf("unable to decode response: %w", err)
return errors.Wrap(err, "unable to decode response")
}
if err := t.Expiration.UnmarshalText([]byte(response.Expire)); err != nil {
return fmt.Errorf("unable to parse jwt expiration: %w", err)
return errors.Wrap(err, "unable to parse jwt expiration")
}
t.Token = response.Token
@ -261,7 +263,7 @@ func (t *JWTTransport) RoundTrip(req *http.Request) (*http.Response, error) {
if err != nil {
/*we had an error (network error for example, or 401 because token is refused), reset the token ?*/
t.Token = ""
return resp, fmt.Errorf("performing jwt auth: %w", err)
return resp, errors.Wrapf(err, "performing jwt auth")
}
log.Debugf("resp-jwt: %d", resp.StatusCode)

2
pkg/apiclient/auth_service.go
View file

@ -21,6 +21,7 @@ type enrollRequest struct {
}
func (s *AuthService) UnregisterWatcher(ctx context.Context) (*Response, error) {
u := fmt.Sprintf("%s/watchers", s.client.URLPrefix)
req, err := s.client.NewRequest(http.MethodDelete, u, nil)
if err != nil {
@ -35,6 +36,7 @@ func (s *AuthService) UnregisterWatcher(ctx context.Context) (*Response, error)
}
func (s *AuthService) RegisterWatcher(ctx context.Context, registration models.WatcherRegistrationRequest) (*Response, error) {
u := fmt.Sprintf("%s/watchers", s.client.URLPrefix)
req, err := s.client.NewRequest(http.MethodPost, u, &registration)

5
pkg/apiclient/auth_service_test.go
View file

@ -10,12 +10,11 @@ import (
"net/url"
"testing"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/models"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
)
type BasicMockPayload struct {

7
pkg/apiclient/client.go
View file

@ -11,6 +11,7 @@ import (
"net/url"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/pkg/errors"
)
var (
@ -124,9 +125,9 @@ func RegisterClient(config *Config, client *http.Client) (*ApiClient, error) {
/*if we have http status, return it*/
if err != nil {
if resp != nil && resp.Response != nil {
return nil, fmt.Errorf("api register (%s) http %s: %w", c.BaseURL, resp.Response.Status, err)
return nil, errors.Wrapf(err, "api register (%s) http %s : %s", c.BaseURL, resp.Response.Status, err)
}
return nil, fmt.Errorf("api register (%s): %w", c.BaseURL, err)
return nil, errors.Wrapf(err, "api register (%s) : %s", c.BaseURL, err)
}
return c, nil
@ -165,7 +166,7 @@ func CheckResponse(r *http.Response) error {
if err == nil && data != nil {
err := json.Unmarshal(data, errorResponse)
if err != nil {
return fmt.Errorf("http code %d, invalid body: %w", r.StatusCode, err)
return errors.Wrapf(err, "http code %d, invalid body", r.StatusCode)
}
} else {
errorResponse.Message = new(string)

4
pkg/apiclient/client_http_test.go
View file

@ -8,9 +8,9 @@ import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/stretchr/testify/assert"
)
func TestNewRequestInvalid(t *testing.T) {

3
pkg/apiclient/client_test.go
View file

@ -9,9 +9,10 @@ import (
"runtime"
"testing"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
)

7
pkg/apiclient/decisions_service.go
View file

@ -6,15 +6,14 @@ import (
"fmt"
"net/http"
qs "github.com/google/go-querystring/query"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/pkg/ptr"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/modelscapi"
"github.com/crowdsecurity/crowdsec/pkg/types"
qs "github.com/google/go-querystring/query"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
type DecisionsService service

7
pkg/apiclient/decisions_service_test.go
View file

@ -8,15 +8,14 @@ import (
"reflect"
"testing"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/pkg/ptr"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/modelscapi"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDecisionsList(t *testing.T) {

8
pkg/apiclient/decisions_sync_service.go
View file

@ -5,9 +5,9 @@ import (
"fmt"
"net/http"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
type DecisionDeleteService service
@ -18,12 +18,12 @@ func (d *DecisionDeleteService) Add(ctx context.Context, deletedDecisions *model
u := fmt.Sprintf("%s/decisions/delete", d.client.URLPrefix)
req, err := d.client.NewRequest(http.MethodPost, u, &deletedDecisions)
if err != nil {
return nil, nil, fmt.Errorf("while building request: %w", err)
return nil, nil, errors.Wrap(err, "while building request")
}
resp, err := d.client.Do(ctx, req, &response)
if err != nil {
return nil, resp, fmt.Errorf("while performing request: %w", err)
return nil, resp, errors.Wrap(err, "while performing request")
}
if resp.Response.StatusCode != http.StatusOK {
log.Warnf("Decisions delete response : http %s", resp.Response.Status)

5
pkg/apiclient/signal.go
View file

@ -8,6 +8,7 @@ import (
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/pkg/errors"
)
type SignalService service
@ -18,12 +19,12 @@ func (s *SignalService) Add(ctx context.Context, signals *models.AddSignalsReque
u := fmt.Sprintf("%s/signals", s.client.URLPrefix)
req, err := s.client.NewRequest(http.MethodPost, u, &signals)
if err != nil {
return nil, nil, fmt.Errorf("while building request: %w", err)
return nil, nil, errors.Wrap(err, "while building request")
}
resp, err := s.client.Do(ctx, req, &response)
if err != nil {
return nil, resp, fmt.Errorf("while performing request: %w", err)
return nil, resp, errors.Wrap(err, "while performing request")
}
if resp.Response.StatusCode != http.StatusOK {
log.Warnf("Signal push response : http %s", resp.Response.Status)

84
pkg/apiserver/apic.go
View file

@ -33,8 +33,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/types"
)
const (
// delta values must be smaller than the interval
var (
pullIntervalDefault = time.Hour * 2
pullIntervalDelta = 5 * time.Minute
pushIntervalDefault = time.Second * 10
@ -72,12 +71,7 @@ type apic struct {
// randomDuration returns a duration value between d-delta and d+delta
func randomDuration(d time.Duration, delta time.Duration) time.Duration {
ret := d + time.Duration(rand.Int63n(int64(2*delta))) - delta
// ticker interval must be > 0 (nanoseconds)
if ret <= 0 {
return 1
}
return ret
return time.Duration(float64(d) + float64(delta)*(-1.0+2.0*rand.Float64()))
}
func (a *apic) FetchScenariosListFromDB() ([]string, error) {
@ -828,6 +822,80 @@ func (a *apic) Pull() error {
}
}
func (a *apic) GetMetrics() (*models.Metrics, error) {
metric := &models.Metrics{
ApilVersion: ptr.Of(version.String()),
Machines: make([]*models.MetricsAgentInfo, 0),
Bouncers: make([]*models.MetricsBouncerInfo, 0),
}
machines, err := a.dbClient.ListMachines()
if err != nil {
return metric, err
}
bouncers, err := a.dbClient.ListBouncers()
if err != nil {
return metric, err
}
var lastpush string
for _, machine := range machines {
if machine.LastPush == nil {
lastpush = time.Time{}.String()
} else {
lastpush = machine.LastPush.String()
}
m := &models.MetricsAgentInfo{
Version: machine.Version,
Name: machine.MachineId,
LastUpdate: machine.UpdatedAt.String(),
LastPush: lastpush,
}
metric.Machines = append(metric.Machines, m)
}
for _, bouncer := range bouncers {
m := &models.MetricsBouncerInfo{
Version: bouncer.Version,
CustomName: bouncer.Name,
Name: bouncer.Type,
LastPull: bouncer.LastPull.String(),
}
metric.Bouncers = append(metric.Bouncers, m)
}
return metric, nil
}
func (a *apic) SendMetrics(stop chan (bool)) {
defer trace.CatchPanic("lapi/metricsToAPIC")
ticker := time.NewTicker(a.metricsIntervalFirst)
log.Infof("Start send metrics to CrowdSec Central API (interval: %s once, then %s)", a.metricsIntervalFirst.Round(time.Second), a.metricsInterval)
for {
metrics, err := a.GetMetrics()
if err != nil {
log.Errorf("unable to get metrics (%s), will retry", err)
}
_, _, err = a.apiClient.Metrics.Add(context.Background(), metrics)
if err != nil {
log.Errorf("capi metrics: failed: %s", err)
} else {
log.Infof("capi metrics: metrics sent successfully")
}
select {
case <-stop:
return
case <-ticker.C:
ticker.Reset(a.metricsInterval)
case <-a.metricsTomb.Dying(): // if one apic routine is dying, do we kill the others?
a.pullTomb.Kill(nil)
a.pushTomb.Kill(nil)
return
}
}
}
func (a *apic) Shutdown() {
a.pushTomb.Kill(nil)
a.pullTomb.Kill(nil)

145
pkg/apiserver/apic_metrics.go
View file

@ -1,145 +0,0 @@
package apiserver
import (
"context"
"time"
log "github.com/sirupsen/logrus"
"golang.org/x/exp/slices"
"github.com/crowdsecurity/go-cs-lib/pkg/ptr"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/models"
)
func (a *apic) GetMetrics() (*models.Metrics, error) {
machines, err := a.dbClient.ListMachines()
if err != nil {
return nil, err
}
machinesInfo := make([]*models.MetricsAgentInfo, len(machines))
for i, machine := range machines {
machinesInfo[i] = &models.MetricsAgentInfo{
Version: machine.Version,
Name: machine.MachineId,
LastUpdate: machine.UpdatedAt.String(),
LastPush: ptr.OrEmpty(machine.LastPush).String(),
}
}
bouncers, err := a.dbClient.ListBouncers()
if err != nil {
return nil, err
}
bouncersInfo := make([]*models.MetricsBouncerInfo, len(bouncers))
for i, bouncer := range bouncers {
bouncersInfo[i] = &models.MetricsBouncerInfo{
Version: bouncer.Version,
CustomName: bouncer.Name,
Name: bouncer.Type,
LastPull: bouncer.LastPull.String(),
}
}
return &models.Metrics{
ApilVersion: ptr.Of(version.String()),
Machines: machinesInfo,
Bouncers: bouncersInfo,
}, nil
}
func (a *apic) fetchMachineIDs() ([]string, error) {
machines, err := a.dbClient.ListMachines()
if err != nil {
return nil, err
}
ret := make([]string, len(machines))
for i, machine := range machines {
ret[i] = machine.MachineId
}
// sorted slices are required for the slices.Equal comparison
slices.Sort(ret)
return ret, nil
}
// SendMetrics sends metrics to the API server until it receives a stop signal.
//
// Metrics are sent at start, then at the randomized metricsIntervalFirst,
// then at regular metricsInterval. If a change is detected in the list
// of machines, the next metrics are sent immediately.
func (a *apic) SendMetrics(stop chan (bool)) {
defer trace.CatchPanic("lapi/metricsToAPIC")
// verify the list of machines every <checkInt> interval
const checkInt = 20 * time.Second
// intervals must always be > 0
metInts := []time.Duration{1, a.metricsIntervalFirst, a.metricsInterval}
log.Infof("Start send metrics to CrowdSec Central API (interval: %s once, then %s)",
metInts[1].Round(time.Second), metInts[2])
count := -1
nextMetInt := func() time.Duration {
if count < len(metInts)-1 {
count++
}
return metInts[count]
}
// store the list of machine IDs to compare
// with the next list
machineIDs := []string{}
reloadMachineIDs := func() {
ids, err := a.fetchMachineIDs()
if err != nil {
log.Debugf("unable to get machines (%s), will retry", err)
return
}
machineIDs = ids
}
checkTicker := time.NewTicker(checkInt)
metTicker := time.NewTicker(nextMetInt())
for {
select {
case <-stop:
checkTicker.Stop()
metTicker.Stop()
return
case <-checkTicker.C:
oldIDs := machineIDs
reloadMachineIDs()
if !slices.Equal(oldIDs, machineIDs) {
log.Infof("capi metrics: machines changed, immediate send")
metTicker.Reset(1)
}
case <-metTicker.C:
metrics, err := a.GetMetrics()
if err != nil {
log.Errorf("unable to get metrics (%s), will retry", err)
}
log.Info("capi metrics: sending")
_, _, err = a.apiClient.Metrics.Add(context.Background(), metrics)
if err != nil {
log.Errorf("capi metrics: failed: %s", err)
}
metTicker.Reset(nextMetInt())
case <-a.metricsTomb.Dying(): // if one apic routine is dying, do we kill the others?
checkTicker.Stop()
metTicker.Stop()
a.pullTomb.Kill(nil)
a.pushTomb.Kill(nil)
return
}
}
}

101
pkg/apiserver/apic_metrics_test.go
View file

@ -1,101 +0,0 @@
package apiserver
import (
"context"
"fmt"
"net/url"
"testing"
"time"
"github.com/jarcoal/httpmock"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
)
func TestAPICSendMetrics(t *testing.T) {
tests := []struct {
name string
duration time.Duration
expectedCalls int
setUp func(*apic)
metricsInterval time.Duration
}{
{
name: "basic",
duration: time.Millisecond * 30,
metricsInterval: time.Millisecond * 5,
expectedCalls: 5,
setUp: func(api *apic) {},
},
{
name: "with some metrics",
duration: time.Millisecond * 30,
metricsInterval: time.Millisecond * 5,
expectedCalls: 5,
setUp: func(api *apic) {
api.dbClient.Ent.Machine.Delete().ExecX(context.Background())
api.dbClient.Ent.Machine.Create().
SetMachineId("1234").
SetPassword(testPassword.String()).
SetIpAddress("1.2.3.4").
SetScenarios("crowdsecurity/test").
SetLastPush(time.Time{}).
SetUpdatedAt(time.Time{}).
ExecX(context.Background())
api.dbClient.Ent.Bouncer.Delete().ExecX(context.Background())
api.dbClient.Ent.Bouncer.Create().
SetIPAddress("1.2.3.6").
SetName("someBouncer").
SetAPIKey("foobar").
SetRevoked(false).
SetLastPull(time.Time{}).
ExecX(context.Background())
},
},
}
httpmock.RegisterResponder("POST", "http://api.crowdsec.net/api/metrics/", httpmock.NewBytesResponder(200, []byte{}))
httpmock.Activate()
defer httpmock.Deactivate()
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
url, err := url.ParseRequestURI("http://api.crowdsec.net/")
require.NoError(t, err)
apiClient, err := apiclient.NewDefaultClient(
url,
"/api",
fmt.Sprintf("crowdsec/%s", version.String()),
nil,
)
require.NoError(t, err)
api := getAPIC(t)
api.pushInterval = time.Millisecond
api.pushIntervalFirst = time.Millisecond
api.apiClient = apiClient
api.metricsInterval = tc.metricsInterval
api.metricsIntervalFirst = tc.metricsInterval
tc.setUp(api)
stop := make(chan bool)
httpmock.ZeroCallCounters()
go api.SendMetrics(stop)
time.Sleep(tc.duration)
stop <- true
info := httpmock.GetCallCountInfo()
noResponderCalls := info["NO_RESPONDER"]
responderCalls := info["POST http://api.crowdsec.net/api/metrics/"]
assert.LessOrEqual(t, absDiff(tc.expectedCalls, responderCalls), 2)
assert.Zero(t, noResponderCalls)
})
}
}

84
pkg/apiserver/apic_test.go
View file

@ -1057,6 +1057,90 @@ func TestAPICPush(t *testing.T) {
}
}
func TestAPICSendMetrics(t *testing.T) {
tests := []struct {
name string
duration time.Duration
expectedCalls int
setUp func(*apic)
metricsInterval time.Duration
}{
{
name: "basic",
duration: time.Millisecond * 30,
metricsInterval: time.Millisecond * 5,
expectedCalls: 5,
setUp: func(api *apic) {},
},
{
name: "with some metrics",
duration: time.Millisecond * 30,
metricsInterval: time.Millisecond * 5,
expectedCalls: 5,
setUp: func(api *apic) {
api.dbClient.Ent.Machine.Delete().ExecX(context.Background())
api.dbClient.Ent.Machine.Create().
SetMachineId("1234").
SetPassword(testPassword.String()).
SetIpAddress("1.2.3.4").
SetScenarios("crowdsecurity/test").
SetLastPush(time.Time{}).
SetUpdatedAt(time.Time{}).
ExecX(context.Background())
api.dbClient.Ent.Bouncer.Delete().ExecX(context.Background())
api.dbClient.Ent.Bouncer.Create().
SetIPAddress("1.2.3.6").
SetName("someBouncer").
SetAPIKey("foobar").
SetRevoked(false).
SetLastPull(time.Time{}).
ExecX(context.Background())
},
},
}
httpmock.RegisterResponder("POST", "http://api.crowdsec.net/api/metrics/", httpmock.NewBytesResponder(200, []byte{}))
httpmock.Activate()
defer httpmock.Deactivate()
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
url, err := url.ParseRequestURI("http://api.crowdsec.net/")
require.NoError(t, err)
apiClient, err := apiclient.NewDefaultClient(
url,
"/api",
fmt.Sprintf("crowdsec/%s", version.String()),
nil,
)
require.NoError(t, err)
api := getAPIC(t)
api.pushInterval = time.Millisecond
api.pushIntervalFirst = time.Millisecond
api.apiClient = apiClient
api.metricsInterval = tc.metricsInterval
api.metricsIntervalFirst = tc.metricsInterval
tc.setUp(api)
stop := make(chan bool)
httpmock.ZeroCallCounters()
go api.SendMetrics(stop)
time.Sleep(tc.duration)
stop <- true
info := httpmock.GetCallCountInfo()
noResponderCalls := info["NO_RESPONDER"]
responderCalls := info["POST http://api.crowdsec.net/api/metrics/"]
assert.LessOrEqual(t, absDiff(tc.expectedCalls, responderCalls), 2)
assert.Zero(t, noResponderCalls)
})
}
}
func TestAPICPull(t *testing.T) {
api := getAPIC(t)
tests := []struct {

28
pkg/apiserver/apiserver.go
View file

@ -9,18 +9,9 @@ import (
"net"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/gin-gonic/gin"
"github.com/go-co-op/gocron"
"github.com/golang-jwt/jwt/v4"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"gopkg.in/natefinch/lumberjack.v2"
"gopkg.in/tomb.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
@ -31,6 +22,13 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/gin-gonic/gin"
"github.com/go-co-op/gocron"
"github.com/golang-jwt/jwt/v4"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"gopkg.in/natefinch/lumberjack.v2"
"gopkg.in/tomb.v2"
)
var (
@ -118,7 +116,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) {
logFile := ""
if config.LogMedia == "file" {
logFile = filepath.Join(config.LogDir, "crowdsec_api.log")
logFile = fmt.Sprintf("%s/crowdsec_api.log", config.LogDir)
}
if log.GetLevel() < log.DebugLevel {
@ -164,7 +162,15 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) {
if config.CompressLogs != nil {
_compress = *config.CompressLogs
}
/*cf. https://github.com/natefinch/lumberjack/issues/82
let's create the file beforehand w/ the right perms */
// check if file exists
_, err := os.Stat(logFile)
// create file if not exists, purposefully ignore errors
if os.IsNotExist(err) {
file, _ := os.OpenFile(logFile, os.O_RDWR|os.O_CREATE, 0600)
file.Close()
}
LogOutput := &lumberjack.Logger{
Filename: logFile,
MaxSize: _maxsize, //megabytes

6
pkg/apiserver/controllers/v1/controller.go
View file

@ -2,15 +2,17 @@ package v1
import (
"context"
"fmt"
"net"
//"github.com/crowdsecurity/crowdsec/pkg/apiserver/controllers"
middlewares "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/csplugin"
"github.com/crowdsecurity/crowdsec/pkg/csprofiles"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/pkg/errors"
)
type Controller struct {
@ -46,7 +48,7 @@ func New(cfg *ControllerV1Config) (*Controller, error) {
profiles, err := csprofiles.NewProfile(cfg.ProfilesCfg)
if err != nil {
return &Controller{}, fmt.Errorf("failed to compile profiles: %w", err)
return &Controller{}, errors.Wrapf(err, "failed to compile profiles")
}
v1 := &Controller{

10
pkg/apiserver/middlewares/v1/api_key.go
View file

@ -3,7 +3,7 @@ package v1
import (
"crypto/rand"
"crypto/sha512"
"encoding/base64"
"encoding/hex"
"fmt"
"net/http"
"strings"
@ -15,11 +15,9 @@ import (
log "github.com/sirupsen/logrus"
)
const (
var (
APIKeyHeader = "X-Api-Key"
bouncerContextKey = "bouncer_info"
// max allowed by bcrypt 72 = 54 bytes in base64
dummyAPIKeySize = 54
)
type APIKey struct {
@ -33,7 +31,7 @@ func GenerateAPIKey(n int) (string, error) {
if _, err := rand.Read(bytes); err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(bytes), nil
return hex.EncodeToString(bytes), nil
}
func NewAPIKey(dbClient *database.Client) *APIKey {
@ -84,7 +82,7 @@ func (a *APIKey) MiddlewareFunc() gin.HandlerFunc {
if err != nil && strings.Contains(err.Error(), "bouncer not found") {
//Because we have a valid cert, automatically create the bouncer in the database if it does not exist
//Set a random API key, but it will never be used
apiKey, err := GenerateAPIKey(dummyAPIKeySize)
apiKey, err := GenerateAPIKey(64)
if err != nil {
log.WithFields(log.Fields{
"ip": c.ClientIP(),

2
pkg/apiserver/middlewares/v1/jwt.go
View file

@ -81,7 +81,7 @@ func (j *JWT) Authenticator(c *gin.Context) (interface{}, error) {
//Machine was not found, let's create it
log.Printf("machine %s not found, create it", machineID)
//let's use an apikey as the password, doesn't matter in this case (generatePassword is only available in cscli)
pwd, err := GenerateAPIKey(dummyAPIKeySize)
pwd, err := GenerateAPIKey(64)
if err != nil {
log.WithFields(log.Fields{
"ip": c.ClientIP(),

7
pkg/apiserver/middlewares/v1/tls_auth.go
View file

@ -12,6 +12,7 @@ import (
"time"
"github.com/gin-gonic/gin"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"golang.org/x/crypto/ocsp"
)
@ -175,9 +176,9 @@ func (ta *TLSAuth) isInvalid(cert *x509.Certificate, issuer *x509.Certificate) (
}
revoked, err := ta.isRevoked(cert, issuer)
if err != nil {
//Fail securely, if we can't check the revocation status, let's consider the cert invalid
//Fail securely, if we can't check the revokation status, let's consider the cert invalid
//We may change this in the future based on users feedback, but this seems the most sensible thing to do
return true, fmt.Errorf("could not check for client certification revocation status: %w", err)
return true, errors.Wrap(err, "could not check for client certification revokation status")
}
return revoked, nil
@ -230,7 +231,7 @@ func (ta *TLSAuth) ValidateCert(c *gin.Context) (bool, string, error) {
revoked, err := ta.isInvalid(clientCert, c.Request.TLS.VerifiedChains[0][1])
if err != nil {
ta.logger.Errorf("TLSAuth: error checking if client certificate is revoked: %s", err)
return false, "", fmt.Errorf("could not check for client certification revokation status: %w", err)
return false, "", errors.Wrap(err, "could not check for client certification revokation status")
}
if revoked {
return false, "", fmt.Errorf("client certificate is revoked")

3
pkg/csconfig/common.go
View file

@ -4,6 +4,7 @@ import (
"fmt"
"path/filepath"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
@ -38,7 +39,7 @@ func (c *Config) LoadCommon() error {
}
*k, err = filepath.Abs(*k)
if err != nil {
return fmt.Errorf("failed to get absolute path of '%s': %w", *k, err)
return errors.Wrapf(err, "failed to get absolute path of '%s'", *k)
}
}

4
pkg/csconfig/config_paths.go
View file

@ -3,6 +3,8 @@ package csconfig
import (
"fmt"
"path/filepath"
"github.com/pkg/errors"
)
type ConfigurationPaths struct {
@ -48,7 +50,7 @@ func (c *Config) LoadConfigurationPaths() error {
}
*k, err = filepath.Abs(*k)
if err != nil {
return fmt.Errorf("failed to get absolute path of '%s': %w", *k, err)
return errors.Wrapf(err, "failed to get absolute path of '%s'", *k)
}
}

4
pkg/csconfig/profiles.go
View file

@ -2,13 +2,13 @@ package csconfig
import (
"bytes"
"errors"
"fmt"
"io"
"github.com/crowdsecurity/go-cs-lib/pkg/yamlpatch"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/pkg/errors"
"gopkg.in/yaml.v2"
)
@ -53,7 +53,7 @@ func (c *LocalApiServerCfg) LoadProfiles() error {
if errors.Is(err, io.EOF) {
break
}
return fmt.Errorf("while decoding %s: %w", c.ProfilesPath, err)
return errors.Wrapf(err, "while decoding %s", c.ProfilesPath)
}
c.Profiles = append(c.Profiles, &t)
}

12
pkg/csplugin/broker.go
View file

@ -2,7 +2,6 @@ package csplugin
import (
"context"
"errors"
"fmt"
"io"
"os"
@ -15,6 +14,7 @@ import (
"github.com/Masterminds/sprig/v3"
"github.com/google/uuid"
plugin "github.com/hashicorp/go-plugin"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
@ -83,10 +83,10 @@ func (pb *PluginBroker) Init(pluginCfg *csconfig.PluginCfg, profileConfigs []*cs
pb.pluginProcConfig = pluginCfg
pb.pluginsTypesToDispatch = make(map[string]struct{})
if err := pb.loadConfig(configPaths.NotificationDir); err != nil {
return fmt.Errorf("while loading plugin config: %w", err)
return errors.Wrap(err, "while loading plugin config")
}
if err := pb.loadPlugins(configPaths.PluginDir); err != nil {
return fmt.Errorf("while loading plugin: %w", err)
return errors.Wrap(err, "while loading plugin")
}
pb.watcher = PluginWatcher{}
pb.watcher.Init(pb.pluginConfigByName, pb.alertsByPluginName)
@ -268,7 +268,7 @@ func (pb *PluginBroker) loadPlugins(path string) error {
data = []byte(csstring.StrictExpand(string(data), os.LookupEnv))
_, err = pluginClient.Configure(context.Background(), &protobufs.Config{Config: data})
if err != nil {
return fmt.Errorf("while configuring %s: %w", pc.Name, err)
return errors.Wrapf(err, "while configuring %s", pc.Name)
}
log.Infof("registered plugin %s", pc.Name)
pb.notificationPluginByName[pc.Name] = pluginClient
@ -354,7 +354,7 @@ func ParsePluginConfigFile(path string) ([]PluginConfig, error) {
parsedConfigs := make([]PluginConfig, 0)
yamlFile, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("while opening %s: %w", path, err)
return parsedConfigs, errors.Wrapf(err, "while opening %s", path)
}
dec := yaml.NewDecoder(yamlFile)
dec.SetStrict(true)
@ -365,7 +365,7 @@ func ParsePluginConfigFile(path string) ([]PluginConfig, error) {
if errors.Is(err, io.EOF) {
break
}
return nil, fmt.Errorf("while decoding %s got error %s", path, err)
return []PluginConfig{}, fmt.Errorf("while decoding %s got error %s", path, err)
}
// if the yaml document is empty, skip
if reflect.DeepEqual(pc, PluginConfig{}) {

6
pkg/csplugin/broker_suite_test.go
View file

@ -14,6 +14,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
type PluginSuite struct {
suite.Suite
@ -31,10 +32,12 @@ type PluginSuite struct {
pluginBroker *PluginBroker
}
func TestPluginSuite(t *testing.T) {
suite.Run(t, new(PluginSuite))
}
func (s *PluginSuite) SetupSuite() {
var err error
@ -54,12 +57,14 @@ func (s *PluginSuite) SetupSuite() {
require.NoError(t, err, "while building dummy plugin")
}
func (s *PluginSuite) TearDownSuite() {
t := s.T()
err := os.RemoveAll(s.buildDir)
require.NoError(t, err)
}
func copyFile(src string, dst string) error {
s, err := os.Open(src)
if err != nil {
@ -94,6 +99,7 @@ func (s *PluginSuite) TearDownTest() {
s.TearDownSubTest()
}
func (s *PluginSuite) SetupSubTest() {
var err error
t := s.T()

13
pkg/csplugin/broker_test.go
View file

@ -22,6 +22,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/models"
)
func (s *PluginSuite) permissionSetter(perm os.FileMode) func(*testing.T) {
return func(t *testing.T) {
err := os.Chmod(s.pluginBinary, perm)
@ -29,28 +30,30 @@ func (s *PluginSuite) permissionSetter(perm os.FileMode) func(*testing.T) {
}
}
func (s *PluginSuite) readconfig() PluginConfig {
func (s *PluginSuite) readconfig() (PluginConfig) {
var config PluginConfig
t := s.T()
orig, err := os.ReadFile(s.pluginConfig)
require.NoError(t, err, "unable to read config file %s", s.pluginConfig)
require.NoError(t, err,"unable to read config file %s", s.pluginConfig)
err = yaml.Unmarshal(orig, &config)
require.NoError(t, err, "unable to unmarshal config file")
require.NoError(t, err,"unable to unmarshal config file")
return config
}
func (s *PluginSuite) writeconfig(config PluginConfig) {
t := s.T()
data, err := yaml.Marshal(&config)
require.NoError(t, err, "unable to marshal config file")
require.NoError(t, err,"unable to marshal config file")
err = os.WriteFile(s.pluginConfig, data, 0644)
require.NoError(t, err, "unable to write config file %s", s.pluginConfig)
require.NoError(t, err,"unable to write config file %s", s.pluginConfig)
}
func (s *PluginSuite) TestBrokerInit() {
tests := []struct {
name string

3
pkg/csplugin/listfiles.go
View file

@ -13,9 +13,10 @@ func listFilesAtPath(path string) ([]string, error) {
return nil, err
}
for _, file := range files {
if !file.IsDir() {
if ! file.IsDir() {
filePaths = append(filePaths, filepath.Join(path, file.Name()))
}
}
return filePaths, nil
}

3
pkg/csplugin/notifier.go
View file

@ -4,10 +4,9 @@ import (
"context"
"fmt"
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
plugin "github.com/hashicorp/go-plugin"
"google.golang.org/grpc"
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
)
type Notifier interface {

11
pkg/csplugin/utils.go
View file

@ -3,7 +3,6 @@
package csplugin
import (
"errors"
"fmt"
"io/fs"
"math"
@ -14,6 +13,8 @@ import (
"strconv"
"strings"
"syscall"
"github.com/pkg/errors"
)
func CheckCredential(uid int, gid int) *syscall.SysProcAttr {
@ -34,7 +35,7 @@ func (pb *PluginBroker) CreateCmd(binaryPath string) (*exec.Cmd, error) {
}
cmd.SysProcAttr, err = getProcessAttr(pb.pluginProcConfig.User, pb.pluginProcConfig.Group)
if err != nil {
return nil, fmt.Errorf("while getting process attributes: %w", err)
return nil, errors.Wrap(err, "while getting process attributes")
}
cmd.SysProcAttr.Credential.NoSetGroups = true
}
@ -104,17 +105,17 @@ func pluginIsValid(path string) error {
// check if it exists
if details, err = os.Stat(path); err != nil {
return fmt.Errorf("plugin at %s does not exist: %w", path, err)
return errors.Wrap(err, fmt.Sprintf("plugin at %s does not exist", path))
}
// check if it is owned by current user
currentUser, err := user.Current()
if err != nil {
return fmt.Errorf("while getting current user: %w", err)
return errors.Wrap(err, "while getting current user")
}
currentUID, err := getUID(currentUser.Username)
if err != nil {
return fmt.Errorf("while looking up the current uid: %w", err)
return errors.Wrap(err, "while looking up the current uid")
}
stat := details.Sys().(*syscall.Stat_t)
if stat.Uid != currentUID {

35
pkg/csplugin/utils_windows.go
View file

@ -13,6 +13,7 @@ import (
"syscall"
"unsafe"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"golang.org/x/sys/windows"
)
@ -53,38 +54,38 @@ func CheckPerms(path string) error {
systemSid, err := windows.CreateWellKnownSid(windows.WELL_KNOWN_SID_TYPE(windows.WinLocalSystemSid))
if err != nil {
return fmt.Errorf("while creating SYSTEM well known sid: %w", err)
return errors.Wrap(err, "while creating SYSTEM well known sid")
}
adminSid, err := windows.CreateWellKnownSid(windows.WELL_KNOWN_SID_TYPE(windows.WinBuiltinAdministratorsSid))
if err != nil {
return fmt.Errorf("while creating built-in Administrators well known sid: %w", err)
return errors.Wrap(err, "while creating built-in Administrators well known sid")
}
currentUser, err := user.Current()
if err != nil {
return fmt.Errorf("while getting current user: %w", err)
return errors.Wrap(err, "while getting current user")
}
currentUserSid, _, _, err := windows.LookupSID("", currentUser.Username)
if err != nil {
return fmt.Errorf("while looking up current user sid: %w", err)
return errors.Wrap(err, "while looking up current user sid")
}
sd, err := windows.GetNamedSecurityInfo(path, windows.SE_FILE_OBJECT, windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION)
if err != nil {
return fmt.Errorf("while getting owner security info: %w", err)
return errors.Wrap(err, "while getting owner security info")
}
if !sd.IsValid() {
return fmt.Errorf("security descriptor is invalid")
return errors.New("security descriptor is invalid")
}
owner, _, err := sd.Owner()
if err != nil {
return fmt.Errorf("while getting owner: %w", err)
return errors.Wrap(err, "while getting owner")
}
if !owner.IsValid() {
return fmt.Errorf("owner is invalid")
return errors.New("owner is invalid")
}
if !owner.Equals(systemSid) && !owner.Equals(currentUserSid) && !owner.Equals(adminSid) {
@ -93,7 +94,7 @@ func CheckPerms(path string) error {
dacl, _, err := sd.DACL()
if err != nil {
return fmt.Errorf("while getting DACL: %w", err)
return errors.Wrap(err, "while getting DACL")
}
if dacl == nil {
@ -101,7 +102,7 @@ func CheckPerms(path string) error {
}
if err != nil {
return fmt.Errorf("while looking up current user sid: %w", err)
return errors.Wrap(err, "while looking up current user sid")
}
rs := reflect.ValueOf(dacl).Elem()
@ -123,7 +124,7 @@ func CheckPerms(path string) error {
ace := &AccessAllowedAce{}
ret, _, _ := procGetAce.Call(uintptr(unsafe.Pointer(dacl)), uintptr(i), uintptr(unsafe.Pointer(&ace)))
if ret == 0 {
return fmt.Errorf("while getting ACE: %w", windows.GetLastError())
return errors.Wrap(windows.GetLastError(), "while getting ACE")
}
log.Debugf("ACE %d: %+v\n", i, ace)
@ -161,14 +162,14 @@ func getProcessAtr() (*syscall.SysProcAttr, error) {
err := windows.OpenProcessToken(proc, windows.TOKEN_DUPLICATE|windows.TOKEN_ADJUST_DEFAULT|
windows.TOKEN_QUERY|windows.TOKEN_ASSIGN_PRIMARY|windows.TOKEN_ADJUST_GROUPS|windows.TOKEN_ADJUST_PRIVILEGES, &procToken)
if err != nil {
return nil, fmt.Errorf("while opening process token: %w", err)
return nil, errors.Wrapf(err, "while opening process token")
}
defer procToken.Close()
err = windows.DuplicateTokenEx(procToken, 0, nil, windows.SecurityImpersonation,
windows.TokenPrimary, &token)
if err != nil {
return nil, fmt.Errorf("while duplicating token: %w", err)
return nil, errors.Wrapf(err, "while duplicating token")
}
//Remove all privileges from the token
@ -176,7 +177,7 @@ func getProcessAtr() (*syscall.SysProcAttr, error) {
err = windows.AdjustTokenPrivileges(token, true, nil, 0, nil, nil)
if err != nil {
return nil, fmt.Errorf("while adjusting token privileges: %w", err)
return nil, errors.Wrapf(err, "while adjusting token privileges")
}
//Run the plugin as a medium integrity level process
@ -194,7 +195,7 @@ func getProcessAtr() (*syscall.SysProcAttr, error) {
(*byte)(unsafe.Pointer(tml)), tml.Size())
if err != nil {
token.Close()
return nil, fmt.Errorf("while setting token information: %w", err)
return nil, errors.Wrapf(err, "while setting token information")
}
return &windows.SysProcAttr{
@ -208,7 +209,7 @@ func (pb *PluginBroker) CreateCmd(binaryPath string) (*exec.Cmd, error) {
cmd := exec.Command(binaryPath)
cmd.SysProcAttr, err = getProcessAtr()
if err != nil {
return nil, fmt.Errorf("while getting process attributes: %w", err)
return nil, errors.Wrap(err, "while getting process attributes")
}
return cmd, err
}
@ -228,7 +229,7 @@ func pluginIsValid(path string) error {
// check if it exists
if _, err = os.Stat(path); err != nil {
return fmt.Errorf("plugin at %s does not exist", path)
return errors.Wrap(err, fmt.Sprintf("plugin at %s does not exist", path))
}
// check if it is owned by root

3
pkg/csplugin/watcher.go
View file

@ -4,10 +4,9 @@ import (
"sync"
"time"
"github.com/crowdsecurity/crowdsec/pkg/models"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
"github.com/crowdsecurity/crowdsec/pkg/models"
)
/*

17
pkg/csplugin/watcher_test.go
View file

@ -7,12 +7,9 @@ import (
"testing"
"time"
"github.com/stretchr/testify/require"
"gopkg.in/tomb.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/cstest"
"github.com/crowdsecurity/crowdsec/pkg/models"
"gopkg.in/tomb.v2"
"gotest.tools/v3/assert"
)
var ctx = context.Background()
@ -67,7 +64,7 @@ func TestPluginWatcherInterval(t *testing.T) {
ct, cancel := context.WithTimeout(ctx, time.Microsecond)
defer cancel()
err := listenChannelWithTimeout(ct, pw.PluginEvents)
cstest.RequireErrorContains(t, err, "context deadline exceeded")
assert.ErrorContains(t, err, "context deadline exceeded")
resetTestTomb(&testTomb, &pw)
testTomb = tomb.Tomb{}
pw.Start(&testTomb)
@ -75,7 +72,7 @@ func TestPluginWatcherInterval(t *testing.T) {
ct, cancel = context.WithTimeout(ctx, time.Millisecond*5)
defer cancel()
err = listenChannelWithTimeout(ct, pw.PluginEvents)
require.NoError(t, err)
assert.NilError(t, err)
resetTestTomb(&testTomb, &pw)
// This is to avoid the int complaining
}
@ -99,7 +96,7 @@ func TestPluginAlertCountWatcher(t *testing.T) {
ct, cancel := context.WithTimeout(ctx, time.Second)
defer cancel()
err := listenChannelWithTimeout(ct, pw.PluginEvents)
cstest.RequireErrorContains(t, err, "context deadline exceeded")
assert.ErrorContains(t, err, "context deadline exceeded")
// Channel won't contain any events since threshold is not crossed.
resetWatcherAlertCounter(&pw)
@ -107,7 +104,7 @@ func TestPluginAlertCountWatcher(t *testing.T) {
ct, cancel = context.WithTimeout(ctx, time.Second)
defer cancel()
err = listenChannelWithTimeout(ct, pw.PluginEvents)
cstest.RequireErrorContains(t, err, "context deadline exceeded")
assert.ErrorContains(t, err, "context deadline exceeded")
// Channel will contain an event since threshold is crossed.
resetWatcherAlertCounter(&pw)
@ -115,6 +112,6 @@ func TestPluginAlertCountWatcher(t *testing.T) {
ct, cancel = context.WithTimeout(ctx, time.Second)
defer cancel()
err = listenChannelWithTimeout(ct, pw.PluginEvents)
require.NoError(t, err)
assert.NilError(t, err)
resetTestTomb(&testTomb, &pw)
}

9
pkg/csprofiles/csprofiles.go
View file

@ -6,13 +6,12 @@ import (
"github.com/antonmedv/expr"
"github.com/antonmedv/expr/vm"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
type Runtime struct {
@ -48,10 +47,10 @@ func NewProfile(profilesCfg []*csconfig.ProfileCfg) ([]*Runtime, error) {
runtime.DebugFilters = make([]*exprhelpers.ExprDebugger, len(profile.Filters))
runtime.Cfg = profile
if runtime.Cfg.OnSuccess != "" && runtime.Cfg.OnSuccess != "continue" && runtime.Cfg.OnSuccess != "break" {
return []*Runtime{}, fmt.Errorf("invalid 'on_success' for '%s': %s", profile.Name, runtime.Cfg.OnSuccess)
return []*Runtime{}, errors.Wrapf(err, "invalid 'on_success' for '%s' : %s", profile.Name, runtime.Cfg.OnSuccess)
}
if runtime.Cfg.OnFailure != "" && runtime.Cfg.OnFailure != "continue" && runtime.Cfg.OnFailure != "break" && runtime.Cfg.OnFailure != "apply" {
return []*Runtime{}, fmt.Errorf("invalid 'on_failure' for '%s' : %s", profile.Name, runtime.Cfg.OnFailure)
return []*Runtime{}, errors.Wrapf(err, "invalid 'on_failure' for '%s' : %s", profile.Name, runtime.Cfg.OnFailure)
}
for fIdx, filter := range profile.Filters {

7
pkg/csprofiles/csprofiles_test.go
View file

@ -5,11 +5,10 @@ import (
"reflect"
"testing"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/models"
"gotest.tools/v3/assert"
)
var (
@ -96,7 +95,7 @@ func TestNewProfile(t *testing.T) {
}
profile, _ := NewProfile(profilesCfg)
fmt.Printf("expected : %+v | result : %+v", test.expectedNbProfile, len(profile))
require.Len(t, profile, test.expectedNbProfile)
assert.Equal(t, test.expectedNbProfile, len(profile))
})
}
}
@ -200,7 +199,7 @@ func TestEvaluateProfile(t *testing.T) {
t.Errorf("EvaluateProfile() got1 = %v, want %v", got1, tt.expectedMatchStatus)
}
if tt.expectedDuration != "" {
require.Equal(t, tt.expectedDuration, *got[0].Duration, "The two durations should be the same")
assert.Equal(t, tt.expectedDuration, *got[0].Duration, "The two durations should be the same")
}
})
}

3
pkg/cticlient/example/fire.go
View file

@ -44,9 +44,6 @@ func main() {
}
for _, item := range items {
if item.State == "refused" {
continue
}
banDuration := time.Until(item.Expiration.Time)
allItems = append(allItems, []string{
item.Ip,

Some files were not shown because too many files have changed in this diff Show more