beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Revert "Merge remote-tracking branch 'origin' into coraza_poc_acquis"

Browse source 
This reverts commit 7098e971c7, reversing
changes made to 13512891e4.
This commit is contained in:
alteredCoder 2023-07-04 18:46:20 +02:00
parent 7098e971c7
commit 84b6570554
244 changed files with 2068 additions and 4722 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
.github/workflows/bats-hub.yml vendored
View file

@ -15,7 +15,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.20.4"]
name: "Build + tests"
runs-on: ubuntu-latest
@ -27,26 +27,40 @@ jobs:
sudo chmod +w /etc/machine-id
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: "Check out CrowdSec repository"
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: true
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: Cache Go modules
uses: actions/cache@v3
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: "Install bats dependencies"
env:
GOBIN: /usr/local/bin
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd
go install github.com/mikefarah/yq/v4@latest
go install github.com/cloudflare/cfssl/cmd/cfssl@master
go install github.com/cloudflare/cfssl/cmd/cfssljson@master
- name: "Build crowdsec and fixture"
run: make bats-clean bats-build bats-fixture BUILD_STATIC=1
run: make bats-clean bats-build bats-fixture
- name: "Run hub tests"
run: make bats-test-hub

28
.github/workflows/bats-mysql.yml vendored
View file

@ -14,7 +14,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.20.4"]
name: "Build + tests"
runs-on: ubuntu-latest
@ -34,27 +34,41 @@ jobs:
sudo chmod +w /etc/machine-id
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: "Check out CrowdSec repository"
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: true
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: Cache Go modules
uses: actions/cache@v3
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: "Install bats dependencies"
env:
GOBIN: /usr/local/bin
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd
go install github.com/mikefarah/yq/v4@latest
go install github.com/cloudflare/cfssl/cmd/cfssl@master
go install github.com/cloudflare/cfssl/cmd/cfssljson@master
- name: "Build crowdsec and fixture"
run: |
make clean bats-build bats-fixture BUILD_STATIC=1
make clean bats-build bats-fixture
env:
DB_BACKEND: mysql
MYSQL_HOST: 127.0.0.1

28
.github/workflows/bats-postgres.yml vendored
View file

@ -10,7 +10,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.20.4"]
name: "Build + tests"
runs-on: ubuntu-latest
@ -35,27 +35,41 @@ jobs:
sudo chmod +w /etc/machine-id
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: "Check out CrowdSec repository"
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: true
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: Cache Go modules
uses: actions/cache@v3
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: "Install bats dependencies"
env:
GOBIN: /usr/local/bin
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd
go install github.com/mikefarah/yq/v4@latest
go install github.com/cloudflare/cfssl/cmd/cfssl@master
go install github.com/cloudflare/cfssl/cmd/cfssljson@master
- name: "Build crowdsec and fixture (DB_BACKEND: pgx)"
run: |
make clean bats-build bats-fixture BUILD_STATIC=1
make clean bats-build bats-fixture
env:
DB_BACKEND: pgx
PGHOST: 127.0.0.1

28
.github/workflows/bats-sqlite-coverage.yml vendored
View file

@ -11,7 +11,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.20.4"]
name: "Build + tests"
runs-on: ubuntu-latest
@ -24,27 +24,41 @@ jobs:
sudo chmod +w /etc/machine-id
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: "Check out CrowdSec repository"
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: true
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: Cache Go modules
uses: actions/cache@v3
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: "Install bats dependencies"
env:
GOBIN: /usr/local/bin
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd
go install github.com/mikefarah/yq/v4@latest
go install github.com/cloudflare/cfssl/cmd/cfssl@master
go install github.com/cloudflare/cfssl/cmd/cfssljson@master
- name: "Build crowdsec and fixture"
run: |
make clean bats-build bats-fixture BUILD_STATIC=1
make clean bats-build bats-fixture
- name: "Run tests"
run: make bats-test

35
.github/workflows/cache-cleanup.yaml vendored
View file

@ -1,35 +0,0 @@
# https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#managing-caches
name: cleanup caches by a branch
on:
pull_request:
types:
- closed
jobs:
cleanup:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v3
- name: Cleanup
run: |
gh extension install actions/gh-actions-cache
REPO=${{ github.repository }}
BRANCH="refs/pull/${{ github.event.pull_request.number }}/merge"
echo "Fetching list of cache key"
cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH | cut -f 1 )
## Setting this to not fail the workflow while deleting cache keys.
set +e
echo "Deleting caches..."
for cacheKey in $cacheKeysForPR
do
gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm
done
echo "Done"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

23
.github/workflows/ci-windows-build-msi.yml vendored
View file

@ -23,27 +23,38 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.20.4"]
name: Build
runs-on: windows-2019
steps:
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: Check out code into the Go module directory
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: false
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: Cache Go modules
uses: actions/cache@v3
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: Build
run: make windows_installer BUILD_RE2_WASM=1
run: make windows_installer
- name: Upload MSI
uses: actions/upload-artifact@v3
with:

12
.github/workflows/docker-tests.yml vendored
View file

@ -30,6 +30,18 @@ jobs:
with:
config: .github/buildkit.toml
# - name: "Build flavor: full"
# uses: docker/build-push-action@v4
# with:
# context: .
# file: ./Dockerfile
# tags: crowdsecurity/crowdsec:test
# target: full
# platforms: linux/amd64
# load: true
# cache-from: type=gha
# cache-to: type=gha,mode=min
- name: "Build flavor: slim"
uses: docker/build-push-action@v4
with:

23
.github/workflows/go-tests-windows.yml vendored
View file

@ -22,28 +22,39 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.20.4"]
name: "Build + tests"
runs-on: windows-2022
steps:
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: Check out CrowdSec repository
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: false
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: Cache Go modules
uses: actions/cache@v3
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: Build
run: |
make build BUILD_RE2_WASM=1
make build
- name: Run tests
run: |

31
.github/workflows/go-tests.yml vendored
View file

@ -34,7 +34,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.20.4"]
name: "Build + tests"
runs-on: ubuntu-latest
@ -110,30 +110,35 @@ jobs:
steps:
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: Check out CrowdSec repository
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: false
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: Cache Go modules
uses: actions/cache@v3
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: Build and run tests, static
- name: Build and run tests
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev
go install github.com/ory/go-acc@v0.2.8
go install github.com/kyoh86/richgo@v0.3.10
set -o pipefail
make build BUILD_STATIC=1
make go-acc | richgo testfilter
- name: Run tests again, dynamic
run: |
make clean build
make build
make go-acc | richgo testfilter
- name: Upload unit coverage to Codecov

27
.github/workflows/release_publish-package.yml vendored
View file

@ -14,32 +14,41 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.20.4"]
name: Build and upload binary package
runs-on: ubuntu-latest
steps:
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: Check out code into the Go module directory
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: false
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: Cache Go modules
uses: actions/cache@v3
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: Build the binaries
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev
make vendor release BUILD_STATIC=1
run: make release
- name: Upload to release
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
tag_name="${GITHUB_REF##*/}"
hub release edit -a crowdsec-release.tgz -a vendor.tgz -m "" "$tag_name"
hub release edit -a crowdsec-release.tgz -m "" "$tag_name"

9
.gitignore vendored
View file

@ -15,9 +15,6 @@
*.test
*.cover
# Test dependencies
test/tools/*
# VMs used for dev/test
.vagrant
@ -33,10 +30,8 @@ test/coverage/*
*.swp
*.swo
# Dependencies are not vendored by default, but a tarball is created by "make vendor"
# and provided in the release. Used by freebsd, gentoo, etc.
vendor/
vendor.tgz
# Dependency directories (remove the comment below to include it)
# vendor/
# crowdsec binaries
cmd/crowdsec-cli/cscli

2
.golangci.yml
View file

@ -65,6 +65,7 @@ linters:
# - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers
# - bidichk # Checks for dangerous unicode character sequences
# - decorder # check declaration order and count of types, constants, variables and functions
# - depguard # Go linter that checks if package imports are in a list of acceptable packages
# - dupword # checks for duplicate words in the source code
# - durationcheck # check for two durations multiplied together
# - errcheck # Errcheck is a program for checking for unchecked errors in go programs. These unchecked errors can be critical bugs in some cases
@ -124,7 +125,6 @@ linters:
- thelper # thelper detects golang test helpers without t.Helper() call and checks the consistency of test helpers
- wastedassign # wastedassign finds wasted assignment statements.
- wrapcheck # Checks that errors returned from external packages are wrapped
- depguard # Go linter that checks if package imports are in a list of acceptable packages
#
# Recommended? (requires some work)

20
Dockerfile
View file

@ -1,31 +1,33 @@
# vim: set ft=dockerfile:
ARG GOVERSION=1.20.5
ARG GOVERSION=1.20.4
FROM golang:${GOVERSION}-alpine AS build
WORKDIR /go/src/crowdsec
# We like to choose the release of re2 to use, and Alpine does not ship a static version anyway.
COPY . .
# Alpine does not ship a static version of re2, we can build it ourselves
# Later versions require 'abseil', which is likewise not available in its static form
ENV RE2_VERSION=2023-03-01
# wizard.sh requires GNU coreutils
RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils pkgconfig && \
RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils icu-static re2-dev pkgconfig && \
wget https://github.com/google/re2/archive/refs/tags/${RE2_VERSION}.tar.gz && \
tar -xzf ${RE2_VERSION}.tar.gz && \
cd re2-${RE2_VERSION} && \
make && \
make install && \
echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
go install github.com/mikefarah/yq/v4@v4.34.1
COPY . .
RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
cd - && \
make clean release DOCKER_BUILD=1 && \
cd crowdsec-v* && \
./wizard.sh --docker-mode && \
cd - >/dev/null && \
cscli hub update && \
cscli collections install crowdsecurity/linux && \
cscli parsers install crowdsecurity/whitelists
cscli parsers install crowdsecurity/whitelists && \
go install github.com/mikefarah/yq/v4@v4.31.2
# In case we need to remove agents here..
# cscli machines list -o json | yq '.[].machineId' | xargs -r cscli machines delete

30
Dockerfile.debian
View file

@ -1,41 +1,32 @@
# vim: set ft=dockerfile:
ARG GOVERSION=1.20.5
ARG GOVERSION=1.20.4
FROM golang:${GOVERSION}-bookworm AS build
FROM golang:${GOVERSION}-bullseye AS build
WORKDIR /go/src/crowdsec
COPY . .
ENV DEBIAN_FRONTEND=noninteractive
ENV DEBCONF_NOWARNINGS="yes"
# We like to choose the release of re2 to use, the debian version is usually older.
ENV RE2_VERSION=2023-03-01
# wizard.sh requires GNU coreutils
RUN apt-get update && \
apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata && \
wget https://github.com/google/re2/archive/refs/tags/${RE2_VERSION}.tar.gz && \
tar -xzf ${RE2_VERSION}.tar.gz && \
cd re2-${RE2_VERSION} && \
make && \
make install && \
apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata libre2-dev && \
echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
go install github.com/mikefarah/yq/v4@v4.34.1
COPY . .
RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
make clean release DOCKER_BUILD=1 && \
cd crowdsec-v* && \
./wizard.sh --docker-mode && \
cd - >/dev/null && \
cscli hub update && \
cscli collections install crowdsecurity/linux && \
cscli parsers install crowdsecurity/whitelists
cscli parsers install crowdsecurity/whitelists && \
go install github.com/mikefarah/yq/v4@v4.31.2
# In case we need to remove agents here..
# cscli machines list -o json | yq '.[].machineId' | xargs -r cscli machines delete
FROM debian:bookworm-slim as slim
FROM debian:bullseye-slim as slim
ENV DEBIAN_FRONTEND=noninteractive
ENV DEBCONF_NOWARNINGS="yes"
@ -53,6 +44,9 @@ RUN apt-get update && \
mkdir -p /staging/var/lib/crowdsec && \
mkdir -p /var/lib/crowdsec/data
RUN echo "deb http://deb.debian.org/debian bullseye-backports main" >> /etc/apt/sources.list \
&& apt-get update && apt-get install -t bullseye-backports -y libsystemd0
COPY --from=build /go/bin/yq /usr/local/bin/yq
COPY --from=build /etc/crowdsec /staging/etc/crowdsec
COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec

99
Makefile
View file

@ -1,36 +1,8 @@
include mk/platform.mk
include mk/gmsl
# By default, this build requires the C++ re2 library to be installed.
#
# Debian/Ubuntu: apt install libre2-dev
# Fedora/CentOS: dnf install re2-devel
# FreeBSD: pkg install re2
# Alpine: apk add re2-dev
# Windows: choco install re2
# MacOS: brew install re2
# To build without re2, run "make BUILD_RE2_WASM=1"
# The WASM version is slower and introduces a short delay when starting a process
# (including cscli) so it is not recommended for production use.
BUILD_RE2_WASM ?= 0
# To build static binaries, run "make BUILD_STATIC=1".
# On some platforms, this requires additional packages
# (e.g. glibc-static and libstdc++-static on fedora, centos.. which are on the powertools/crb repository).
# If the static build fails at the link stage, it might be because the static library is not provided
# for your distribution (look for libre2.a). See the Dockerfile for an example of how to build it.
BUILD_STATIC ?= 0
# List of plugins to build
PLUGINS ?= $(patsubst ./plugins/notifications/%,%,$(wildcard ./plugins/notifications/*))
# Can be overriden, if you can deal with the consequences
BUILD_REQUIRE_GO_MAJOR ?= 1
BUILD_REQUIRE_GO_MINOR ?= 20
#--------------------------------------
GOCMD = go
GOTEST = $(GOCMD) test
@ -38,6 +10,8 @@ BUILD_CODENAME ?= alphaga
CROWDSEC_FOLDER = ./cmd/crowdsec
CSCLI_FOLDER = ./cmd/crowdsec-cli/
PLUGINS ?= $(patsubst ./plugins/notifications/%,%,$(wildcard ./plugins/notifications/*))
PLUGINS_DIR = ./plugins/notifications
CROWDSEC_BIN = crowdsec$(EXT)
@ -48,14 +22,8 @@ RELDIR = crowdsec-$(BUILD_VERSION)
GO_MODULE_NAME = github.com/crowdsecurity/crowdsec
# Check if a given value is considered truthy and returns "0" or "1".
# A truthy value is one of the following: "1", "yes", or "true", case-insensitive.
#
# Usage:
# ifeq ($(call bool,$(FOO)),1)
# $(info Let's foo)
# endif
bool = $(if $(filter $(call lc, $1),1 yes true),1,0)
# see if we have libre2-dev installed for C++ optimizations
RE2_CHECK := $(shell pkg-config --libs re2 2>/dev/null)
#--------------------------------------
#
@ -78,30 +46,13 @@ endif
GO_TAGS := netgo,osusergo,sqlite_omit_load_extension
ifeq ($(call bool,$(BUILD_RE2_WASM)),0)
ifeq ($(PKG_CONFIG),)
$(error "pkg-config is not available. Please install pkg-config.")
endif
ifeq ($(RE2_CHECK),)
# we could detect the platform and suggest the command to install
RE2_FAIL := "libre2-dev is not installed, please install it or set BUILD_RE2_WASM=1 to use the WebAssembly version"
else
ifneq (,$(RE2_CHECK))
# += adds a space that we don't want
GO_TAGS := $(GO_TAGS),re2_cgo
LD_OPTS_VARS += -X '$(GO_MODULE_NAME)/pkg/cwversion.Libre2=C++'
endif
endif
ifeq ($(call bool,$(BUILD_STATIC)),1)
BUILD_TYPE = static
EXTLDFLAGS := -extldflags '-static'
else
BUILD_TYPE = dynamic
EXTLDFLAGS :=
endif
export LD_OPTS=-ldflags "-s -w $(EXTLDFLAGS) $(LD_OPTS_VARS)" \
export LD_OPTS=-ldflags "-s -w -extldflags '-static' $(LD_OPTS_VARS)" \
-trimpath -tags $(GO_TAGS)
ifneq (,$(TEST_COVERAGE))
@ -113,15 +64,12 @@ endif
.PHONY: build
build: pre-build goversion crowdsec cscli plugins
# Sanity checks and build information
.PHONY: pre-build
pre-build:
$(info Building $(BUILD_VERSION) ($(BUILD_TAG)) $(BUILD_TYPE) for $(GOOS)/$(GOARCH))
ifneq (,$(RE2_FAIL))
$(error $(RE2_FAIL))
ifdef BUILD_STATIC
$(warning WARNING: The BUILD_STATIC variable is deprecated and has no effect. Builds are static by default since v1.5.0.)
endif
$(info Building $(BUILD_VERSION) ($(BUILD_TAG)) for $(GOOS)/$(GOARCH))
ifneq (,$(RE2_CHECK))
$(info Using C++ regexp library)
else
@ -165,7 +113,6 @@ testclean: bats-clean
@$(RM) pkg/cwhub/install $(WIN_IGNORE_ERR)
@$(RM) pkg/types/example.txt $(WIN_IGNORE_ERR)
# for the tests with localstack
export AWS_ENDPOINT_FORCE=http://localhost:4566
export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
@ -173,18 +120,15 @@ export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
testenv:
@echo 'NOTE: You need Docker, docker-compose and run "make localstack" in a separate shell ("make localstack-stop" to terminate it)'
# run the tests with localstack
.PHONY: test
test: testenv goversion
$(GOTEST) $(LD_OPTS) ./...
# run the tests with localstack and coverage
.PHONY: go-acc
go-acc: testenv goversion
go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models -- $(LD_OPTS) | \
sed 's/ *coverage:.*of statements in.*//'
# mock AWS services
.PHONY: localstack
localstack:
docker-compose -f test/localstack/docker-compose.yml up
@ -193,27 +137,13 @@ localstack:
localstack-stop:
docker-compose -f test/localstack/docker-compose.yml down
# list of plugins that contain go.mod
PLUGIN_VENDOR = $(foreach plugin,$(PLUGINS),$(shell if [ -f $(PLUGINS_DIR)/$(plugin)/go.mod ]; then echo $(PLUGINS_DIR)/$(plugin); fi))
# build vendor.tgz to be distributed with the release
.PHONY: vendor
vendor:
$(foreach plugin_dir,$(PLUGIN_VENDOR), \
cd $(plugin_dir) >/dev/null && \
$(GOCMD) mod vendor && \
cd - >/dev/null; \
@echo "Vendoring dependencies"
@$(GOCMD) mod vendor
@$(foreach plugin,$(PLUGINS), \
$(MAKE) -C $(PLUGINS_DIR)/$(plugin) vendor $(MAKE_FLAGS); \
)
$(GOCMD) mod vendor
tar -czf vendor.tgz vendor $(foreach plugin_dir,$(PLUGIN_VENDOR),$(plugin_dir)/vendor)
# remove vendor directories and vendor.tgz
.PHONY: vendor-remove
vendor-remove:
$(foreach plugin_dir,$(PLUGIN_VENDOR), \
$(RM) $(plugin_dir)/vendor; \
)
$(RM) vendor vendor.tgz
.PHONY: package
package:
@ -244,16 +174,13 @@ else
@if (Test-Path -Path $(RELDIR)) { echo "$(RELDIR) already exists, abort" ; exit 1 ; }
endif
# build a release tarball
.PHONY: release
release: check_release build package
# build the windows installer
.PHONY: windows_installer
windows_installer: build
@.\make_installer.ps1 -version $(BUILD_VERSION)
# build the chocolatey package
.PHONY: chocolatey
chocolatey: windows_installer
@.\make_chocolatey.ps1 -version $(BUILD_VERSION)

4
azure-pipelines.yml
View file

@ -27,7 +27,7 @@ stages:
- task: GoTool@0
displayName: "Install Go 1.20"
inputs:
version: '1.20.5'
version: '1.20.4'
- pwsh: |
choco install -y make
@ -38,7 +38,7 @@ stages:
pwsh: true
#we are not calling make windows_installer because we want to sign the binaries before they are added to the MSI
script: |
make build BUILD_RE2_WASM=1
make build
- task: AzureKeyVault@2
inputs:
azureSubscription: 'Azure subscription 1(8a93ab40-7e99-445e-ad47-0f6a3e2ef546)'

8
cmd/crowdsec-cli/alerts.go
View file

@ -15,6 +15,7 @@ import (
"github.com/fatih/color"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
@ -154,6 +155,7 @@ var alertTemplate = `
`
func DisplayOneAlert(alert *models.Alert, withDetail bool) error {
if csConfig.Cscli.Output == "human" {
tmpl, err := template.New("alert").Parse(alertTemplate)
@ -209,11 +211,11 @@ func NewAlertsCmd() *cobra.Command {
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
var err error
if err := csConfig.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %w", err)
return errors.Wrap(err, "loading api client")
}
apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL)
if err != nil {
return fmt.Errorf("parsing api url %s: %w", apiURL, err)
return errors.Wrapf(err, "parsing api url %s", apiURL)
}
Client, err = apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Client.Credentials.Login,
@ -224,7 +226,7 @@ func NewAlertsCmd() *cobra.Command {
})
if err != nil {
return fmt.Errorf("new api client: %w", err)
return errors.Wrap(err, "new api client")
}
return nil
},

19
cmd/crowdsec-cli/capi.go
View file

@ -6,11 +6,6 @@ import (
"net/url"
"os"
"github.com/go-openapi/strfmt"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
@ -19,6 +14,11 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/fflag"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
)
const CAPIBaseURL string = "https://api.crowdsec.net/"
@ -31,11 +31,8 @@ func NewCapiCmd() *cobra.Command {
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadAPIServer(); err != nil {
return fmt.Errorf("local API is disabled, please run this command on the local API machine: %w", err)
}
if csConfig.DisableAPI {
return nil
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
return errors.Wrap(err, "Local API is disabled, please run this command on the local API machine")
}
if csConfig.API.Server.OnlineClient == nil {
log.Fatalf("no configuration for Central API in '%s'", *csConfig.FilePath)
@ -136,7 +133,7 @@ func NewCapiStatusCmd() *cobra.Command {
Run: func(cmd *cobra.Command, args []string) {
var err error
if csConfig.API.Server == nil {
log.Fatal("There is no configuration on 'api.server:'")
log.Fatalln("There is no configuration on 'api.server:'")
}
if csConfig.API.Server.OnlineClient == nil {
log.Fatalf("Please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)

42
cmd/crowdsec-cli/config_backup.go
View file

@ -5,9 +5,11 @@ import (
"os"
"path/filepath"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
@ -32,17 +34,17 @@ func backupConfigToDirectory(dirPath string) error {
/*if parent directory doesn't exist, bail out. create final dir with Mkdir*/
parentDir := filepath.Dir(dirPath)
if _, err := os.Stat(parentDir); err != nil {
return fmt.Errorf("while checking parent directory %s existence: %w", parentDir, err)
return errors.Wrapf(err, "while checking parent directory %s existence", parentDir)
}
if err = os.Mkdir(dirPath, 0o700); err != nil {
return fmt.Errorf("while creating %s: %w", dirPath, err)
return errors.Wrapf(err, "while creating %s", dirPath)
}
if csConfig.ConfigPaths.SimulationFilePath != "" {
backupSimulation := filepath.Join(dirPath, "simulation.yaml")
if err = CopyFile(csConfig.ConfigPaths.SimulationFilePath, backupSimulation); err != nil {
return fmt.Errorf("failed copy %s to %s: %w", csConfig.ConfigPaths.SimulationFilePath, backupSimulation, err)
if err = types.CopyFile(csConfig.ConfigPaths.SimulationFilePath, backupSimulation); err != nil {
return errors.Wrapf(err, "failed copy %s to %s", csConfig.ConfigPaths.SimulationFilePath, backupSimulation)
}
log.Infof("Saved simulation to %s", backupSimulation)
@ -54,14 +56,14 @@ func backupConfigToDirectory(dirPath string) error {
*/
if csConfig.Crowdsec != nil && csConfig.Crowdsec.AcquisitionFilePath != "" {
backupAcquisition := filepath.Join(dirPath, "acquis.yaml")
if err = CopyFile(csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil {
return fmt.Errorf("failed copy %s to %s: %s", csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition, err)
if err = types.CopyFile(csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition, err)
}
}
acquisBackupDir := filepath.Join(dirPath, "acquis")
if err = os.Mkdir(acquisBackupDir, 0o700); err != nil {
return fmt.Errorf("error while creating %s: %s", acquisBackupDir, err)
return fmt.Errorf("error while creating %s : %s", acquisBackupDir, err)
}
if csConfig.Crowdsec != nil && len(csConfig.Crowdsec.AcquisitionFiles) > 0 {
@ -73,11 +75,11 @@ func backupConfigToDirectory(dirPath string) error {
targetFname, err := filepath.Abs(filepath.Join(acquisBackupDir, filepath.Base(acquisFile)))
if err != nil {
return fmt.Errorf("while saving %s to %s: %w", acquisFile, acquisBackupDir, err)
return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir)
}
if err = CopyFile(acquisFile, targetFname); err != nil {
return fmt.Errorf("failed copy %s to %s: %w", acquisFile, targetFname, err)
if err = types.CopyFile(acquisFile, targetFname); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err)
}
log.Infof("Saved acquis %s to %s", acquisFile, targetFname)
@ -86,8 +88,8 @@ func backupConfigToDirectory(dirPath string) error {
if ConfigFilePath != "" {
backupMain := fmt.Sprintf("%s/config.yaml", dirPath)
if err = CopyFile(ConfigFilePath, backupMain); err != nil {
return fmt.Errorf("failed copy %s to %s: %s", ConfigFilePath, backupMain, err)
if err = types.CopyFile(ConfigFilePath, backupMain); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", ConfigFilePath, backupMain, err)
}
log.Infof("Saved default yaml to %s", backupMain)
@ -95,8 +97,8 @@ func backupConfigToDirectory(dirPath string) error {
if csConfig.API != nil && csConfig.API.Server != nil && csConfig.API.Server.OnlineClient != nil && csConfig.API.Server.OnlineClient.CredentialsFilePath != "" {
backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath)
if err = CopyFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds); err != nil {
return fmt.Errorf("failed copy %s to %s: %s", csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds, err)
if err = types.CopyFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds, err)
}
log.Infof("Saved online API credentials to %s", backupCAPICreds)
@ -104,8 +106,8 @@ func backupConfigToDirectory(dirPath string) error {
if csConfig.API != nil && csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" {
backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath)
if err = CopyFile(csConfig.API.Client.CredentialsFilePath, backupLAPICreds); err != nil {
return fmt.Errorf("failed copy %s to %s: %s", csConfig.API.Client.CredentialsFilePath, backupLAPICreds, err)
if err = types.CopyFile(csConfig.API.Client.CredentialsFilePath, backupLAPICreds); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Client.CredentialsFilePath, backupLAPICreds, err)
}
log.Infof("Saved local API credentials to %s", backupLAPICreds)
@ -113,20 +115,21 @@ func backupConfigToDirectory(dirPath string) error {
if csConfig.API != nil && csConfig.API.Server != nil && csConfig.API.Server.ProfilesPath != "" {
backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath)
if err = CopyFile(csConfig.API.Server.ProfilesPath, backupProfiles); err != nil {
return fmt.Errorf("failed copy %s to %s: %s", csConfig.API.Server.ProfilesPath, backupProfiles, err)
if err = types.CopyFile(csConfig.API.Server.ProfilesPath, backupProfiles); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Server.ProfilesPath, backupProfiles, err)
}
log.Infof("Saved profiles to %s", backupProfiles)
}
if err = BackupHub(dirPath); err != nil {
return fmt.Errorf("failed to backup hub config: %s", err)
return fmt.Errorf("failed to backup hub config : %s", err)
}
return nil
}
func runConfigBackup(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadHub(); err != nil {
return err
@ -144,6 +147,7 @@ func runConfigBackup(cmd *cobra.Command, args []string) error {
return nil
}
func NewConfigBackupCmd() *cobra.Command {
cmdConfigBackup := &cobra.Command{
Use: `backup "directory"`,

24
cmd/crowdsec-cli/config_restore.go
View file

@ -7,11 +7,13 @@ import (
"os"
"path/filepath"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
@ -36,7 +38,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
backupMain := fmt.Sprintf("%s/config.yaml", dirPath)
if _, err = os.Stat(backupMain); err == nil {
if csConfig.ConfigPaths != nil && csConfig.ConfigPaths.ConfigDir != "" {
if err = CopyFile(backupMain, fmt.Sprintf("%s/config.yaml", csConfig.ConfigPaths.ConfigDir)); err != nil {
if err = types.CopyFile(backupMain, fmt.Sprintf("%s/config.yaml", csConfig.ConfigPaths.ConfigDir)); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", backupMain, csConfig.ConfigPaths.ConfigDir, err)
}
}
@ -49,21 +51,21 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath)
if _, err = os.Stat(backupCAPICreds); err == nil {
if err = CopyFile(backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath); err != nil {
if err = types.CopyFile(backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath, err)
}
}
backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath)
if _, err = os.Stat(backupLAPICreds); err == nil {
if err = CopyFile(backupLAPICreds, csConfig.API.Client.CredentialsFilePath); err != nil {
if err = types.CopyFile(backupLAPICreds, csConfig.API.Client.CredentialsFilePath); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", backupLAPICreds, csConfig.API.Client.CredentialsFilePath, err)
}
}
backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath)
if _, err = os.Stat(backupProfiles); err == nil {
if err = CopyFile(backupProfiles, csConfig.API.Server.ProfilesPath); err != nil {
if err = types.CopyFile(backupProfiles, csConfig.API.Server.ProfilesPath); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", backupProfiles, csConfig.API.Server.ProfilesPath, err)
}
}
@ -104,7 +106,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
backupSimulation := fmt.Sprintf("%s/simulation.yaml", dirPath)
if _, err = os.Stat(backupSimulation); err == nil {
if err = CopyFile(backupSimulation, csConfig.ConfigPaths.SimulationFilePath); err != nil {
if err = types.CopyFile(backupSimulation, csConfig.ConfigPaths.SimulationFilePath); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", backupSimulation, csConfig.ConfigPaths.SimulationFilePath, err)
}
}
@ -121,7 +123,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
if _, err = os.Stat(backupAcquisition); err == nil {
log.Debugf("restoring backup'ed %s", backupAcquisition)
if err = CopyFile(backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath); err != nil {
if err = types.CopyFile(backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath, err)
}
}
@ -132,12 +134,12 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
for _, acquisFile := range acquisFiles {
targetFname, err := filepath.Abs(csConfig.Crowdsec.AcquisitionDirPath + "/" + filepath.Base(acquisFile))
if err != nil {
return fmt.Errorf("while saving %s to %s: %w", acquisFile, targetFname, err)
return errors.Wrapf(err, "while saving %s to %s", acquisFile, targetFname)
}
log.Debugf("restoring %s to %s", acquisFile, targetFname)
if err = CopyFile(acquisFile, targetFname); err != nil {
if err = types.CopyFile(acquisFile, targetFname); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err)
}
}
@ -155,10 +157,10 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
targetFname, err := filepath.Abs(filepath.Join(acquisBackupDir, filepath.Base(acquisFile)))
if err != nil {
return fmt.Errorf("while saving %s to %s: %w", acquisFile, acquisBackupDir, err)
return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir)
}
if err = CopyFile(acquisFile, targetFname); err != nil {
if err = types.CopyFile(acquisFile, targetFname); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err)
}
@ -173,6 +175,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
return nil
}
func runConfigRestore(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
@ -197,6 +200,7 @@ func runConfigRestore(cmd *cobra.Command, args []string) error {
return nil
}
func NewConfigRestoreCmd() *cobra.Command {
cmdConfigRestore := &cobra.Command{
Use: `restore "directory"`,

73
cmd/crowdsec-cli/copyfile.go
View file

@ -1,73 +0,0 @@
package main
import (
"fmt"
"io"
"os"
"path/filepath"
log "github.com/sirupsen/logrus"
)
/*help to copy the file, ioutil doesn't offer the feature*/
func copyFileContents(src, dst string) (err error) {
in, err := os.Open(src)
if err != nil {
return
}
defer in.Close()
out, err := os.Create(dst)
if err != nil {
return
}
defer func() {
cerr := out.Close()
if err == nil {
err = cerr
}
}()
if _, err = io.Copy(out, in); err != nil {
return
}
err = out.Sync()
return
}
/*copy the file, ioutile doesn't offer the feature*/
func CopyFile(sourceSymLink, destinationFile string) (err error) {
sourceFile, err := filepath.EvalSymlinks(sourceSymLink)
if err != nil {
log.Infof("Not a symlink : %s", err)
sourceFile = sourceSymLink
}
sourceFileStat, err := os.Stat(sourceFile)
if err != nil {
return
}
if !sourceFileStat.Mode().IsRegular() {
// cannot copy non-regular files (e.g., directories,
// symlinks, devices, etc.)
return fmt.Errorf("copyFile: non-regular source file %s (%q)", sourceFileStat.Name(), sourceFileStat.Mode().String())
}
destinationFileStat, err := os.Stat(destinationFile)
if err != nil {
if !os.IsNotExist(err) {
return
}
} else {
if !(destinationFileStat.Mode().IsRegular()) {
return fmt.Errorf("copyFile: non-regular destination file %s (%q)", destinationFileStat.Name(), destinationFileStat.Mode().String())
}
if os.SameFile(sourceFileStat, destinationFileStat) {
return
}
}
if err = os.Link(sourceFile, destinationFile); err != nil {
err = copyFileContents(sourceFile, destinationFile)
}
return
}

245
cmd/crowdsec-cli/decisions.go
View file

@ -7,15 +7,19 @@ import (
"fmt"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/fatih/color"
"github.com/go-openapi/strfmt"
"github.com/jszwec/csvutil"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/go-cs-lib/pkg/ptr"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
@ -108,12 +112,12 @@ func NewDecisionsCmd() *cobra.Command {
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %w", err)
return errors.Wrap(err, "loading api client")
}
password := strfmt.Password(csConfig.API.Client.Credentials.Password)
apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL)
if err != nil {
return fmt.Errorf("parsing api url %s: %w", csConfig.API.Client.Credentials.URL, err)
return errors.Wrapf(err, "parsing api url %s", csConfig.API.Client.Credentials.URL)
}
Client, err = apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Client.Credentials.Login,
@ -123,7 +127,7 @@ func NewDecisionsCmd() *cobra.Command {
VersionPrefix: "v1",
})
if err != nil {
return fmt.Errorf("creating api client: %w", err)
return errors.Wrap(err, "creating api client")
}
return nil
},
@ -165,11 +169,11 @@ cscli decisions list -t ban
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
Run: func(cmd *cobra.Command, args []string) {
var err error
/*take care of shorthand options*/
if err = manageCliDecisionAlerts(filter.IPEquals, filter.RangeEquals, filter.ScopeEquals, filter.ValueEquals); err != nil {
return err
if err := manageCliDecisionAlerts(filter.IPEquals, filter.RangeEquals, filter.ScopeEquals, filter.ValueEquals); err != nil {
log.Fatalf("%s", err)
}
filter.ActiveDecisionEquals = new(bool)
*filter.ActiveDecisionEquals = true
@ -185,7 +189,7 @@ cscli decisions list -t ban
days, err := strconv.Atoi(realDuration)
if err != nil {
printHelp(cmd)
return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until)
log.Fatalf("Can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until)
}
*filter.Until = fmt.Sprintf("%d%s", days*24, "h")
}
@ -198,7 +202,7 @@ cscli decisions list -t ban
days, err := strconv.Atoi(realDuration)
if err != nil {
printHelp(cmd)
return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Since)
log.Fatalf("Can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until)
}
*filter.Since = fmt.Sprintf("%d%s", days*24, "h")
}
@ -234,15 +238,13 @@ cscli decisions list -t ban
alerts, _, err := Client.Alerts.List(context.Background(), filter)
if err != nil {
return fmt.Errorf("unable to retrieve decisions: %w", err)
log.Fatalf("Unable to list decisions : %v", err)
}
err = DecisionsToTable(alerts, printMachine)
if err != nil {
return fmt.Errorf("unable to print decisions: %w", err)
log.Fatalf("unable to list decisions : %v", err)
}
return nil
},
}
cmdDecisionsList.Flags().SortFlags = false
@ -286,7 +288,7 @@ cscli decisions add --scope username --value foobar
/*TBD : fix long and example*/
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
Run: func(cmd *cobra.Command, args []string) {
var err error
alerts := models.AddAlertsRequest{}
origin := types.CscliOrigin
@ -301,7 +303,7 @@ cscli decisions add --scope username --value foobar
/*take care of shorthand options*/
if err := manageCliDecisionAlerts(&addIP, &addRange, &addScope, &addValue); err != nil {
return err
log.Fatalf("%s", err)
}
if addIP != "" {
@ -312,7 +314,7 @@ cscli decisions add --scope username --value foobar
addScope = types.Range
} else if addValue == "" {
printHelp(cmd)
return fmt.Errorf("Missing arguments, a value is required (--ip, --range or --scope and --value)")
log.Fatalf("Missing arguments, a value is required (--ip, --range or --scope and --value)")
}
if addReason == "" {
@ -355,11 +357,10 @@ cscli decisions add --scope username --value foobar
_, _, err = Client.Alerts.Add(context.Background(), alerts)
if err != nil {
return err
log.Fatal(err)
}
log.Info("Decision successfully added")
return nil
},
}
@ -400,27 +401,25 @@ cscli decisions delete --id 42
cscli decisions delete --type captcha
`,
/*TBD : refaire le Long/Example*/
PreRunE: func(cmd *cobra.Command, args []string) error {
PreRun: func(cmd *cobra.Command, args []string) {
if delDecisionAll {
return nil
return
}
if *delFilter.ScopeEquals == "" && *delFilter.ValueEquals == "" &&
*delFilter.TypeEquals == "" && *delFilter.IPEquals == "" &&
*delFilter.RangeEquals == "" && *delFilter.ScenarioEquals == "" &&
*delFilter.OriginEquals == "" && delDecisionId == "" {
cmd.Usage()
return fmt.Errorf("at least one filter or --all must be specified")
log.Fatalln("At least one filter or --all must be specified")
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
Run: func(cmd *cobra.Command, args []string) {
var err error
var decisions *models.DeleteDecisionResponse
/*take care of shorthand options*/
if err = manageCliDecisionAlerts(delFilter.IPEquals, delFilter.RangeEquals, delFilter.ScopeEquals, delFilter.ValueEquals); err != nil {
return err
if err := manageCliDecisionAlerts(delFilter.IPEquals, delFilter.RangeEquals, delFilter.ScopeEquals, delFilter.ValueEquals); err != nil {
log.Fatalf("%s", err)
}
if *delFilter.ScopeEquals == "" {
delFilter.ScopeEquals = nil
@ -450,19 +449,18 @@ cscli decisions delete --type captcha
if delDecisionId == "" {
decisions, _, err = Client.Decisions.Delete(context.Background(), delFilter)
if err != nil {
return fmt.Errorf("Unable to delete decisions: %v", err)
log.Fatalf("Unable to delete decisions : %v", err)
}
} else {
if _, err = strconv.Atoi(delDecisionId); err != nil {
return fmt.Errorf("id '%s' is not an integer: %v", delDecisionId, err)
log.Fatalf("id '%s' is not an integer: %v", delDecisionId, err)
}
decisions, _, err = Client.Decisions.DeleteOne(context.Background(), delDecisionId)
if err != nil {
return fmt.Errorf("Unable to delete decision: %v", err)
log.Fatalf("Unable to delete decision : %v", err)
}
}
log.Infof("%s decision(s) deleted", decisions.NbDeleted)
return nil
},
}
@ -480,3 +478,192 @@ cscli decisions delete --type captcha
return cmdDecisionsDelete
}
func NewDecisionsImportCmd() *cobra.Command {
var (
defaultDuration = "4h"
defaultScope = "ip"
defaultType = "ban"
defaultReason = "manual"
importDuration string
importScope string
importReason string
importType string
importFile string
batchSize int
)
var cmdDecisionImport = &cobra.Command{
Use: "import [options]",
Short: "Import decisions from json or csv file",
Long: "expected format :\n" +
"csv : any of duration,origin,reason,scope,type,value, with a header line\n" +
`json : {"duration" : "24h", "origin" : "my-list", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"}`,
DisableAutoGenTag: true,
Example: `decisions.csv :
duration,scope,value
24h,ip,1.2.3.4
cscsli decisions import -i decisions.csv
decisions.json :
[{"duration" : "4h", "scope" : "ip", "type" : "ban", "value" : "1.2.3.4"}]
`,
Run: func(cmd *cobra.Command, args []string) {
if importFile == "" {
log.Fatalf("Please provide a input file containing decisions with -i flag")
}
csvData, err := os.ReadFile(importFile)
if err != nil {
log.Fatalf("unable to open '%s': %s", importFile, err)
}
type decisionRaw struct {
Duration string `csv:"duration,omitempty" json:"duration,omitempty"`
Origin string `csv:"origin,omitempty" json:"origin,omitempty"`
Scenario string `csv:"reason,omitempty" json:"reason,omitempty"`
Scope string `csv:"scope,omitempty" json:"scope,omitempty"`
Type string `csv:"type,omitempty" json:"type,omitempty"`
Value string `csv:"value" json:"value"`
}
var decisionsListRaw []decisionRaw
switch fileFormat := filepath.Ext(importFile); fileFormat {
case ".json":
if err := json.Unmarshal(csvData, &decisionsListRaw); err != nil {
log.Fatalf("unable to unmarshall json: '%s'", err)
}
case ".csv":
if err := csvutil.Unmarshal(csvData, &decisionsListRaw); err != nil {
log.Fatalf("unable to unmarshall csv: '%s'", err)
}
default:
log.Fatalf("file format not supported for '%s'. supported format are 'json' and 'csv'", importFile)
}
decisionsList := make([]*models.Decision, 0)
for i, decisionLine := range decisionsListRaw {
line := i + 2
if decisionLine.Value == "" {
log.Fatalf("please provide a 'value' in your csv line %d", line)
}
/*deal with defaults and cli-override*/
if decisionLine.Duration == "" {
decisionLine.Duration = defaultDuration
log.Debugf("No 'duration' line %d, using default value: '%s'", line, defaultDuration)
}
if importDuration != "" {
decisionLine.Duration = importDuration
log.Debugf("'duration' line %d, using supplied value: '%s'", line, importDuration)
}
decisionLine.Origin = types.CscliImportOrigin
if decisionLine.Scenario == "" {
decisionLine.Scenario = defaultReason
log.Debugf("No 'reason' line %d, using value: '%s'", line, decisionLine.Scenario)
}
if importReason != "" {
decisionLine.Scenario = importReason
log.Debugf("No 'reason' line %d, using supplied value: '%s'", line, importReason)
}
if decisionLine.Type == "" {
decisionLine.Type = defaultType
log.Debugf("No 'type' line %d, using default value: '%s'", line, decisionLine.Type)
}
if importType != "" {
decisionLine.Type = importType
log.Debugf("'type' line %d, using supplied value: '%s'", line, importType)
}
if decisionLine.Scope == "" {
decisionLine.Scope = defaultScope
log.Debugf("No 'scope' line %d, using default value: '%s'", line, decisionLine.Scope)
}
if importScope != "" {
decisionLine.Scope = importScope
log.Debugf("'scope' line %d, using supplied value: '%s'", line, importScope)
}
decision := models.Decision{
Value: ptr.Of(decisionLine.Value),
Duration: ptr.Of(decisionLine.Duration),
Origin: ptr.Of(decisionLine.Origin),
Scenario: ptr.Of(decisionLine.Scenario),
Type: ptr.Of(decisionLine.Type),
Scope: ptr.Of(decisionLine.Scope),
Simulated: new(bool),
}
decisionsList = append(decisionsList, &decision)
}
alerts := models.AddAlertsRequest{}
if batchSize > 0 {
for i := 0; i < len(decisionsList); i += batchSize {
end := i + batchSize
if end > len(decisionsList) {
end = len(decisionsList)
}
decisionBatch := decisionsList[i:end]
importAlert := models.Alert{
CreatedAt: time.Now().UTC().Format(time.RFC3339),
Scenario: ptr.Of(fmt.Sprintf("import %s : %d IPs", importFile, len(decisionBatch))),
Message: ptr.Of(""),
Events: []*models.Event{},
Source: &models.Source{
Scope: ptr.Of(""),
Value: ptr.Of(""),
},
StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
Capacity: ptr.Of(int32(0)),
Simulated: ptr.Of(false),
EventsCount: ptr.Of(int32(len(decisionBatch))),
Leakspeed: ptr.Of(""),
ScenarioHash: ptr.Of(""),
ScenarioVersion: ptr.Of(""),
Decisions: decisionBatch,
}
alerts = append(alerts, &importAlert)
}
} else {
importAlert := models.Alert{
CreatedAt: time.Now().UTC().Format(time.RFC3339),
Scenario: ptr.Of(fmt.Sprintf("import %s : %d IPs", importFile, len(decisionsList))),
Message: ptr.Of(""),
Events: []*models.Event{},
Source: &models.Source{
Scope: ptr.Of(""),
Value: ptr.Of(""),
},
StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
Capacity: ptr.Of(int32(0)),
Simulated: ptr.Of(false),
EventsCount: ptr.Of(int32(len(decisionsList))),
Leakspeed: ptr.Of(""),
ScenarioHash: ptr.Of(""),
ScenarioVersion: ptr.Of(""),
Decisions: decisionsList,
}
alerts = append(alerts, &importAlert)
}
if len(decisionsList) > 1000 {
log.Infof("You are about to add %d decisions, this may take a while", len(decisionsList))
}
_, _, err = Client.Alerts.Add(context.Background(), alerts)
if err != nil {
log.Fatal(err)
}
log.Infof("%d decisions successfully imported", len(decisionsList))
},
}
cmdDecisionImport.Flags().SortFlags = false
cmdDecisionImport.Flags().StringVarP(&importFile, "input", "i", "", "Input file")
cmdDecisionImport.Flags().StringVarP(&importDuration, "duration", "d", "", "Decision duration (ie. 1h,4h,30m)")
cmdDecisionImport.Flags().StringVar(&importScope, "scope", types.Ip, "Decision scope (ie. ip,range,username)")
cmdDecisionImport.Flags().StringVarP(&importReason, "reason", "R", "", "Decision reason (ie. scenario-name)")
cmdDecisionImport.Flags().StringVarP(&importType, "type", "t", "", "Decision type (ie. ban,captcha,throttle)")
cmdDecisionImport.Flags().IntVar(&batchSize, "batch", 0, "Split import in batches of N decisions")
return cmdDecisionImport
}

272
cmd/crowdsec-cli/decisions_import.go
View file

@ -1,272 +0,0 @@
package main
import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"os"
"strings"
"time"
"github.com/jszwec/csvutil"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/go-cs-lib/pkg/ptr"
"github.com/crowdsecurity/go-cs-lib/pkg/slicetools"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
// decisionRaw is only used to unmarshall json/csv decisions
type decisionRaw struct {
Duration string `csv:"duration,omitempty" json:"duration,omitempty"`
Scenario string `csv:"reason,omitempty" json:"reason,omitempty"`
Scope string `csv:"scope,omitempty" json:"scope,omitempty"`
Type string `csv:"type,omitempty" json:"type,omitempty"`
Value string `csv:"value" json:"value"`
}
func parseDecisionList(content []byte, format string) ([]decisionRaw, error) {
ret := []decisionRaw{}
switch format {
case "values":
log.Infof("Parsing values")
scanner := bufio.NewScanner(bytes.NewReader(content))
for scanner.Scan() {
value := strings.TrimSpace(scanner.Text())
ret = append(ret, decisionRaw{Value: value})
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("unable to parse values: '%s'", err)
}
case "json":
log.Infof("Parsing json")
if err := json.Unmarshal(content, &ret); err != nil {
return nil, err
}
case "csv":
log.Infof("Parsing csv")
if err := csvutil.Unmarshal(content, &ret); err != nil {
return nil, fmt.Errorf("unable to parse csv: '%s'", err)
}
default:
return nil, fmt.Errorf("invalid format '%s', expected one of 'json', 'csv', 'values'", format)
}
return ret, nil
}
func runDecisionsImport(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
input, err := flags.GetString("input")
if err != nil {
return err
}
defaultDuration, err := flags.GetString("duration")
if err != nil {
return err
}
if defaultDuration == "" {
return fmt.Errorf("--duration cannot be empty")
}
defaultScope, err := flags.GetString("scope")
if err != nil {
return err
}
if defaultScope == "" {
return fmt.Errorf("--scope cannot be empty")
}
defaultReason, err := flags.GetString("reason")
if err != nil {
return err
}
if defaultReason == "" {
return fmt.Errorf("--reason cannot be empty")
}
defaultType, err := flags.GetString("type")
if err != nil {
return err
}
if defaultType == "" {
return fmt.Errorf("--type cannot be empty")
}
batchSize, err := flags.GetInt("batch")
if err != nil {
return err
}
format, err := flags.GetString("format")
if err != nil {
return err
}
var (
content []byte
fin *os.File
)
// set format if the file has a json or csv extension
if format == "" {
if strings.HasSuffix(input, ".json") {
format = "json"
} else if strings.HasSuffix(input, ".csv") {
format = "csv"
}
}
if format == "" {
return fmt.Errorf("unable to guess format from file extension, please provide a format with --format flag")
}
if input == "-" {
fin = os.Stdin
input = "stdin"
} else {
fin, err = os.Open(input)
if err != nil {
return fmt.Errorf("unable to open %s: %s", input, err)
}
}
content, err = io.ReadAll(fin)
if err != nil {
return fmt.Errorf("unable to read from %s: %s", input, err)
}
decisionsListRaw, err := parseDecisionList(content, format)
if err != nil {
return err
}
decisions := make([]*models.Decision, len(decisionsListRaw))
for i, d := range decisionsListRaw {
if d.Value == "" {
return fmt.Errorf("item %d: missing 'value'", i)
}
if d.Duration == "" {
d.Duration = defaultDuration
log.Debugf("item %d: missing 'duration', using default '%s'", i, defaultDuration)
}
if d.Scenario == "" {
d.Scenario = defaultReason
log.Debugf("item %d: missing 'reason', using default '%s'", i, defaultReason)
}
if d.Type == "" {
d.Type = defaultType
log.Debugf("item %d: missing 'type', using default '%s'", i, defaultType)
}
if d.Scope == "" {
d.Scope = defaultScope
log.Debugf("item %d: missing 'scope', using default '%s'", i, defaultScope)
}
decisions[i] = &models.Decision{
Value: ptr.Of(d.Value),
Duration: ptr.Of(d.Duration),
Origin: ptr.Of(types.CscliImportOrigin),
Scenario: ptr.Of(d.Scenario),
Type: ptr.Of(d.Type),
Scope: ptr.Of(d.Scope),
Simulated: ptr.Of(false),
}
}
alerts := models.AddAlertsRequest{}
for _, chunk := range slicetools.Chunks(decisions, batchSize) {
log.Debugf("Processing chunk of %d decisions", len(chunk))
importAlert := models.Alert{
CreatedAt: time.Now().UTC().Format(time.RFC3339),
Scenario: ptr.Of(fmt.Sprintf("import %s: %d IPs", input, len(chunk))),
Message: ptr.Of(""),
Events: []*models.Event{},
Source: &models.Source{
Scope: ptr.Of(""),
Value: ptr.Of(""),
},
StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
Capacity: ptr.Of(int32(0)),
Simulated: ptr.Of(false),
EventsCount: ptr.Of(int32(len(chunk))),
Leakspeed: ptr.Of(""),
ScenarioHash: ptr.Of(""),
ScenarioVersion: ptr.Of(""),
Decisions: chunk,
}
alerts = append(alerts, &importAlert)
}
if len(decisions) > 1000 {
log.Infof("You are about to add %d decisions, this may take a while", len(decisions))
}
_, _, err = Client.Alerts.Add(context.Background(), alerts)
if err != nil {
return err
}
log.Infof("Imported %d decisions", len(decisions))
return nil
}
func NewDecisionsImportCmd() *cobra.Command {
var cmdDecisionsImport = &cobra.Command{
Use: "import [options]",
Short: "Import decisions from a file or pipe",
Long: "expected format:\n" +
"csv : any of duration,reason,scope,type,value, with a header line\n" +
`json : {"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"}`,
DisableAutoGenTag: true,
Example: `decisions.csv:
duration,scope,value
24h,ip,1.2.3.4
$ cscli decisions import -i decisions.csv
decisions.json:
[{"duration" : "4h", "scope" : "ip", "type" : "ban", "value" : "1.2.3.4"}]
The file format is detected from the extension, but can be forced with the --format option
which is required when reading from standard input.
Raw values, standard input:
$ echo "1.2.3.4" | cscli decisions import -i - --format values
`,
RunE: runDecisionsImport,
}
flags := cmdDecisionsImport.Flags()
flags.SortFlags = false
flags.StringP("input", "i", "", "Input file")
flags.StringP("duration", "d", "4h", "Decision duration: 1h,4h,30m")
flags.String("scope", types.Ip, "Decision scope: ip,range,username")
flags.StringP("reason", "R", "manual", "Decision reason: <scenario-name>")
flags.StringP("type", "t", "ban", "Decision type: ban,captcha,throttle")
flags.Int("batch", 0, "Split import in batches of N decisions")
flags.String("format", "", "Input format: 'json', 'csv' or 'values' (each line is a value, no headers)")
cmdDecisionsImport.MarkFlagRequired("input")
return cmdDecisionsImport
}

6
cmd/crowdsec-cli/lapi.go
View file

@ -9,6 +9,7 @@ import (
"strings"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"golang.org/x/exp/slices"
@ -23,6 +24,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
var LAPIURLPrefix string = "v1"
@ -203,7 +205,7 @@ func NewLapiCmd() *cobra.Command {
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %w", err)
return errors.Wrap(err, "loading api client")
}
return nil
},
@ -438,7 +440,7 @@ cscli lapi context delete --value evt.Line.Src
return cmdContext
}
func detectStaticField(GrokStatics []parser.ExtraField) []string {
func detectStaticField(GrokStatics []types.ExtraField) []string {
ret := make([]string, 0)
for _, static := range GrokStatics {
if static.Parsed != "" {

35
cmd/crowdsec-cli/machines.go
View file

@ -12,6 +12,7 @@ import (
"time"
"github.com/AlecAivazis/survey/v2"
"github.com/enescakir/emoji"
"github.com/fatih/color"
"github.com/go-openapi/strfmt"
"github.com/google/uuid"
@ -84,21 +85,22 @@ func generateID(prefix string) (string, error) {
return prefix + suffix, nil
}
// getLastHeartbeat returns the last heartbeat timestamp of a machine
// and a boolean indicating if the machine is considered active or not.
func getLastHeartbeat(m *ent.Machine) (string, bool) {
if m.LastHeartbeat == nil {
return "-", false
func displayLastHeartBeat(m *ent.Machine, fancy bool) string {
var hbDisplay string
if m.LastHeartbeat != nil {
lastHeartBeat := time.Now().UTC().Sub(*m.LastHeartbeat)
hbDisplay = lastHeartBeat.Truncate(time.Second).String()
if fancy && lastHeartBeat > 2*time.Minute {
hbDisplay = fmt.Sprintf("%s %s", emoji.Warning.String(), lastHeartBeat.Truncate(time.Second).String())
}
} else {
hbDisplay = "-"
if fancy {
hbDisplay = emoji.Warning.String() + " -"
}
}
elapsed := time.Now().UTC().Sub(*m.LastHeartbeat)
hb := elapsed.Truncate(time.Second).String()
if elapsed > 2*time.Minute {
return hb, false
}
return hb, true
return hbDisplay
}
func getAgents(out io.Writer, dbClient *database.Client) error {
@ -128,10 +130,9 @@ func getAgents(out io.Writer, dbClient *database.Client) error {
} else {
validated = "false"
}
hb, _ := getLastHeartbeat(m)
err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb})
err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, displayLastHeartBeat(m, false)})
if err != nil {
return fmt.Errorf("failed to write raw output: %w", err)
return fmt.Errorf("failed to write raw output : %s", err)
}
}
csvwriter.Flush()

6
cmd/crowdsec-cli/machines_table.go
View file

@ -24,11 +24,7 @@ func getAgentsTable(out io.Writer, machines []*ent.Machine) {
validated = emoji.Prohibited.String()
}
hb, active := getLastHeartbeat(m)
if !active {
hb = emoji.Warning.String() + " " + hb
}
t.AddRow(m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb)
t.AddRow(m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, displayLastHeartBeat(m, true))
}
t.Render()

58
cmd/crowdsec-cli/notifications.go
View file

@ -15,6 +15,7 @@ import (
"github.com/fatih/color"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/tomb.v2"
@ -27,12 +28,14 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/csprofiles"
)
type NotificationsCfg struct {
Config csplugin.PluginConfig `json:"plugin_config"`
Profiles []*csconfig.ProfileCfg `json:"associated_profiles"`
ids []uint
}
func NewNotificationsCmd() *cobra.Command {
var cmdNotifications = &cobra.Command{
Use: "notifications [action]",
@ -54,6 +57,7 @@ func NewNotificationsCmd() *cobra.Command {
},
}
cmdNotifications.AddCommand(NewNotificationsListCmd())
cmdNotifications.AddCommand(NewNotificationsInspectCmd())
cmdNotifications.AddCommand(NewNotificationsReinjectCmd())
@ -61,17 +65,18 @@ func NewNotificationsCmd() *cobra.Command {
return cmdNotifications
}
func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
pcfgs := map[string]csplugin.PluginConfig{}
wf := func(path string, info fs.FileInfo, err error) error {
if info == nil {
return fmt.Errorf("error while traversing directory %s: %w", path, err)
return errors.Wrapf(err, "error while traversing directory %s", path)
}
name := filepath.Join(csConfig.ConfigPaths.NotificationDir, info.Name()) //Avoid calling info.Name() twice
if (strings.HasSuffix(name, "yaml") || strings.HasSuffix(name, "yml")) && !(info.IsDir()) {
ts, err := csplugin.ParsePluginConfigFile(name)
if err != nil {
return fmt.Errorf("loading notifification plugin configuration with %s: %w", name, err)
return errors.Wrapf(err, "Loading notifification plugin configuration with %s", name)
}
for _, t := range ts {
pcfgs[t.Name] = t
@ -81,14 +86,14 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
}
if err := filepath.Walk(csConfig.ConfigPaths.NotificationDir, wf); err != nil {
return nil, fmt.Errorf("while loading notifification plugin configuration: %w", err)
return nil, errors.Wrap(err, "Loading notifification plugin configuration")
}
// A bit of a tricky stuf now: reconcile profiles and notification plugins
ncfgs := map[string]NotificationsCfg{}
profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles)
if err != nil {
return nil, fmt.Errorf("while extracting profiles from configuration: %w", err)
return nil, errors.Wrap(err, "Cannot extract profiles from configuration")
}
for profileID, profile := range profiles {
loop:
@ -124,6 +129,7 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
return ncfgs, nil
}
func NewNotificationsListCmd() *cobra.Command {
var cmdNotificationsList = &cobra.Command{
Use: "list",
@ -135,7 +141,7 @@ func NewNotificationsListCmd() *cobra.Command {
RunE: func(cmd *cobra.Command, arg []string) error {
ncfgs, err := getNotificationsConfiguration()
if err != nil {
return fmt.Errorf("can't build profiles configuration: %w", err)
return errors.Wrap(err, "Can't build profiles configuration")
}
if csConfig.Cscli.Output == "human" {
@ -143,14 +149,14 @@ func NewNotificationsListCmd() *cobra.Command {
} else if csConfig.Cscli.Output == "json" {
x, err := json.MarshalIndent(ncfgs, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal notification configuration: %w", err)
return errors.New("failed to marshal notification configuration")
}
fmt.Printf("%s", string(x))
} else if csConfig.Cscli.Output == "raw" {
csvwriter := csv.NewWriter(os.Stdout)
err := csvwriter.Write([]string{"Name", "Type", "Profile name"})
if err != nil {
return fmt.Errorf("failed to write raw header: %w", err)
return errors.Wrap(err, "failed to write raw header")
}
for _, b := range ncfgs {
profilesList := []string{}
@ -159,7 +165,7 @@ func NewNotificationsListCmd() *cobra.Command {
}
err := csvwriter.Write([]string{b.Config.Name, b.Config.Type, strings.Join(profilesList, ", ")})
if err != nil {
return fmt.Errorf("failed to write raw content: %w", err)
return errors.Wrap(err, "failed to write raw content")
}
}
csvwriter.Flush()
@ -171,6 +177,7 @@ func NewNotificationsListCmd() *cobra.Command {
return cmdNotificationsList
}
func NewNotificationsInspectCmd() *cobra.Command {
var cmdNotificationsInspect = &cobra.Command{
Use: "inspect",
@ -188,14 +195,14 @@ func NewNotificationsInspectCmd() *cobra.Command {
pluginName := arg[0]
if pluginName == "" {
return fmt.Errorf("please provide a plugin name to inspect")
errors.New("Please provide a plugin name to inspect")
}
ncfgs, err := getNotificationsConfiguration()
if err != nil {
return fmt.Errorf("can't build profiles configuration: %w", err)
return errors.Wrap(err, "Can't build profiles configuration")
}
if cfg, ok = ncfgs[pluginName]; !ok {
return fmt.Errorf("plugin '%s' does not exist or is not active", pluginName)
return errors.New("The provided plugin name doesn't exist or isn't active")
}
if csConfig.Cscli.Output == "human" || csConfig.Cscli.Output == "raw" {
@ -209,7 +216,7 @@ func NewNotificationsInspectCmd() *cobra.Command {
} else if csConfig.Cscli.Output == "json" {
x, err := json.MarshalIndent(cfg, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal notification configuration: %w", err)
return errors.New("failed to marshal notification configuration")
}
fmt.Printf("%s", string(x))
}
@ -220,6 +227,7 @@ func NewNotificationsInspectCmd() *cobra.Command {
return cmdNotificationsInspect
}
func NewNotificationsReinjectCmd() *cobra.Command {
var remediation bool
var alertOverride string
@ -242,26 +250,26 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
)
if len(args) != 1 {
printHelp(cmd)
return fmt.Errorf("wrong number of argument: there should be one argument")
return errors.New("Wrong number of argument: there should be one argument")
}
//first: get the alert
id, err := strconv.Atoi(args[0])
if err != nil {
return fmt.Errorf("bad alert id %s", args[0])
return errors.New(fmt.Sprintf("bad alert id %s", args[0]))
}
if err := csConfig.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %w", err)
return errors.Wrapf(err, "loading api client")
}
if csConfig.API.Client == nil {
return fmt.Errorf("missing configuration on 'api_client:'")
return errors.New("There is no configuration on 'api_client:'")
}
if csConfig.API.Client.Credentials == nil {
return fmt.Errorf("missing API credentials in '%s'", csConfig.API.Client.CredentialsFilePath)
return errors.New(fmt.Sprintf("Please provide credentials for the API in '%s'", csConfig.API.Client.CredentialsFilePath))
}
apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL)
if err != nil {
return fmt.Errorf("error parsing the URL of the API: %w", err)
return errors.Wrapf(err, "error parsing the URL of the API")
}
client, err := apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Client.Credentials.Login,
@ -271,16 +279,16 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
VersionPrefix: "v1",
})
if err != nil {
return fmt.Errorf("error creating the client for the API: %w", err)
return errors.Wrapf(err, "error creating the client for the API")
}
alert, _, err := client.Alerts.GetByID(context.Background(), id)
if err != nil {
return fmt.Errorf("can't find alert with id %s: %w", args[0], err)
return errors.Wrapf(err, fmt.Sprintf("can't find alert with id %s", args[0]))
}
if alertOverride != "" {
if err = json.Unmarshal([]byte(alertOverride), alert); err != nil {
return fmt.Errorf("can't unmarshal data in the alert flag: %w", err)
return errors.Wrapf(err, "Can't unmarshal the data given in the alert flag")
}
}
if !remediation {
@ -290,7 +298,7 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
// second we start plugins
err = pluginBroker.Init(csConfig.PluginConfig, csConfig.API.Server.Profiles, csConfig.ConfigPaths)
if err != nil {
return fmt.Errorf("can't initialize plugins: %w", err)
return errors.Wrapf(err, "Can't initialize plugins")
}
pluginTomb.Go(func() error {
@ -302,13 +310,13 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles)
if err != nil {
return fmt.Errorf("cannot extract profiles from configuration: %w", err)
return errors.Wrap(err, "Cannot extract profiles from configuration")
}
for id, profile := range profiles {
_, matched, err := profile.EvaluateProfile(alert)
if err != nil {
return fmt.Errorf("can't evaluate profile %s: %w", profile.Cfg.Name, err)
return errors.Wrapf(err, "can't evaluate profile %s", profile.Cfg.Name)
}
if !matched {
log.Infof("The profile %s didn't match", profile.Cfg.Name)
@ -336,7 +344,7 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
}
// time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
pluginTomb.Kill(fmt.Errorf("terminating"))
pluginTomb.Kill(errors.New("terminating"))
pluginTomb.Wait()
return nil
},

3
cmd/crowdsec-cli/scenarios.go
View file

@ -4,6 +4,7 @@ import (
"fmt"
"github.com/fatih/color"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
@ -32,7 +33,7 @@ cscli scenarios remove crowdsecurity/ssh-bf
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("while setting hub branch: %w", err)
return errors.Wrap(err, "while setting hub branch")
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {

11
cmd/crowdsec-cli/support.go
View file

@ -26,6 +26,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
const (
@ -47,14 +48,6 @@ const (
SUPPORT_CROWDSEC_PROFILE_PATH = "config/profiles.yaml"
)
// from https://github.com/acarl005/stripansi
var reStripAnsi = regexp.MustCompile("[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))")
func stripAnsiString(str string) string {
// the byte version doesn't strip correctly
return reStripAnsi.ReplaceAllString(str, "")
}
func collectMetrics() ([]byte, []byte, error) {
log.Info("Collecting prometheus metrics")
err := csConfig.LoadPrometheus()
@ -407,7 +400,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
log.Errorf("Could not add zip entry for %s: %s", filename, err)
continue
}
fw.Write([]byte(stripAnsiString(string(data))))
fw.Write([]byte(types.StripAnsiString(string(data))))
}
err = zipWriter.Close()

6
cmd/crowdsec-cli/utils.go
View file

@ -598,7 +598,7 @@ func RestoreHub(dirPath string) error {
log.Infof("Going to restore local/tainted [%s]", tfile.Name())
sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
if err = CopyFile(sourceFile, destinationFile); err != nil {
if err = types.CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
@ -607,7 +607,7 @@ func RestoreHub(dirPath string) error {
log.Infof("Going to restore local/tainted [%s]", file.Name())
sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
if err = CopyFile(sourceFile, destinationFile); err != nil {
if err = types.CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
@ -657,7 +657,7 @@ func BackupHub(dirPath string) error {
}
clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
if err = CopyFile(v.LocalPath, tfile); err != nil {
if err = types.CopyFile(v.LocalPath, tfile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
}
clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)

11
cmd/crowdsec/api.go
View file

@ -1,7 +1,6 @@
package main
import (
"fmt"
"runtime"
"time"
@ -21,7 +20,7 @@ func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) {
apiServer, err := apiserver.NewServer(cConfig.API.Server)
if err != nil {
return nil, fmt.Errorf("unable to run local API: %w", err)
return nil, errors.Wrap(err, "unable to run local API")
}
if hasPlugins(cConfig.API.Server.Profiles) {
@ -30,27 +29,23 @@ func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) {
if cConfig.PluginConfig == nil && runtime.GOOS != "windows" {
return nil, errors.New("plugins are enabled, but the plugin_config section is missing in the configuration")
}
if cConfig.ConfigPaths.NotificationDir == "" {
return nil, errors.New("plugins are enabled, but config_paths.notification_dir is not defined")
}
if cConfig.ConfigPaths.PluginDir == "" {
return nil, errors.New("plugins are enabled, but config_paths.plugin_dir is not defined")
}
err = pluginBroker.Init(cConfig.PluginConfig, cConfig.API.Server.Profiles, cConfig.ConfigPaths)
if err != nil {
return nil, fmt.Errorf("unable to run plugin broker: %w", err)
return nil, errors.Wrap(err, "unable to run local API")
}
log.Info("initiated plugin broker")
apiServer.AttachPluginBroker(&pluginBroker)
}
err = apiServer.InitController()
if err != nil {
return nil, fmt.Errorf("unable to run local API: %w", err)
return nil, errors.Wrap(err, "unable to run local API")
}
return apiServer, nil

19
cmd/crowdsec/crowdsec.go
View file

@ -3,12 +3,10 @@ package main
import (
"fmt"
"os"
"path/filepath"
"sync"
"time"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
"path/filepath"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
@ -18,28 +16,31 @@ import (
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
)
func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
var err error
// Populate cwhub package tools
if err = cwhub.GetHubIdx(cConfig.Hub); err != nil {
return nil, fmt.Errorf("while loading hub index: %w", err)
if err := cwhub.GetHubIdx(cConfig.Hub); err != nil {
return &parser.Parsers{}, fmt.Errorf("Failed to load hub index : %s", err)
}
// Start loading configs
csParsers := parser.NewParsers()
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
return nil, fmt.Errorf("while loading parsers: %w", err)
return &parser.Parsers{}, fmt.Errorf("Failed to load parsers: %s", err)
}
if err := LoadBuckets(cConfig); err != nil {
return nil, fmt.Errorf("while loading scenarios: %w", err)
return &parser.Parsers{}, fmt.Errorf("Failed to load scenarios: %s", err)
}
if err := LoadAcquisition(cConfig); err != nil {
return nil, fmt.Errorf("while loading acquisition config: %w", err)
return &parser.Parsers{}, fmt.Errorf("Error while loading acquisition config : %s", err)
}
return csParsers, nil
}
@ -117,7 +118,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
aggregated = true
}
if err := acquisition.GetMetrics(dataSources, aggregated); err != nil {
return fmt.Errorf("while fetching prometheus metrics for datasources: %w", err)
return errors.Wrap(err, "while fetching prometheus metrics for datasources.")
}
}

46
cmd/crowdsec/main.go
View file

@ -51,15 +51,12 @@ var (
)
type Flags struct {
ConfigFile string
LogLevelTrace bool
LogLevelDebug bool
LogLevelInfo bool
LogLevelWarn bool
LogLevelError bool
LogLevelFatal bool
ConfigFile string
TraceLevel bool
DebugLevel bool
InfoLevel bool
WarnLevel bool
ErrorLevel bool
PrintVersion bool
SingleFileType string
Labels map[string]string
@ -110,7 +107,7 @@ func LoadAcquisition(cConfig *csconfig.Config) error {
dataSources, err = acquisition.LoadAcquisitionFromDSN(flags.OneShotDSN, flags.Labels, flags.Transform)
if err != nil {
return fmt.Errorf("failed to configure datasource for %s: %w", flags.OneShotDSN, err)
return errors.Wrapf(err, "failed to configure datasource for %s", flags.OneShotDSN)
}
} else {
dataSources, err = acquisition.LoadAcquisitionFromFile(cConfig.Crowdsec)
@ -119,10 +116,6 @@ func LoadAcquisition(cConfig *csconfig.Config) error {
}
}
if len(dataSources) == 0 {
return fmt.Errorf("no datasource enabled")
}
return nil
}
@ -147,14 +140,11 @@ func (l labelsMap) Set(label string) error {
func (f *Flags) Parse() {
flag.StringVar(&f.ConfigFile, "c", csconfig.DefaultConfigPath("config.yaml"), "configuration file")
flag.BoolVar(&f.LogLevelTrace, "trace", false, "set log level to 'trace' (VERY verbose)")
flag.BoolVar(&f.LogLevelDebug, "debug", false, "set log level to 'debug'")
flag.BoolVar(&f.LogLevelInfo, "info", false, "set log level to 'info'")
flag.BoolVar(&f.LogLevelWarn, "warning", false, "set log level to 'warning'")
flag.BoolVar(&f.LogLevelError, "error", false, "set log level to 'error'")
flag.BoolVar(&f.LogLevelFatal, "fatal", false, "set log level to 'fatal'")
flag.BoolVar(&f.TraceLevel, "trace", false, "VERY verbose")
flag.BoolVar(&f.DebugLevel, "debug", false, "print debug-level on stderr")
flag.BoolVar(&f.InfoLevel, "info", false, "print info-level on stderr")
flag.BoolVar(&f.WarnLevel, "warning", false, "print warning-level on stderr")
flag.BoolVar(&f.ErrorLevel, "error", false, "print error-level on stderr")
flag.BoolVar(&f.PrintVersion, "version", false, "display version")
flag.StringVar(&f.OneShotDSN, "dsn", "", "Process a single data source in time-machine")
flag.StringVar(&f.Transform, "transform", "", "expr to apply on the event after acquisition")
@ -182,18 +172,16 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level {
// override from flags
switch {
case f.LogLevelTrace:
case f.TraceLevel:
ret = log.TraceLevel
case f.LogLevelDebug:
case f.DebugLevel:
ret = log.DebugLevel
case f.LogLevelInfo:
case f.InfoLevel:
ret = log.InfoLevel
case f.LogLevelWarn:
case f.WarnLevel:
ret = log.WarnLevel
case f.LogLevelError:
case f.ErrorLevel:
ret = log.ErrorLevel
case f.LogLevelFatal:
ret = log.FatalLevel
default:
}

15
cmd/crowdsec/output.go
View file

@ -7,10 +7,6 @@ import (
"sync"
"time"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
@ -20,6 +16,9 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
func dedupAlerts(alerts []types.RuntimeAlert) ([]*models.Alert, error) {
@ -51,11 +50,11 @@ func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error
alertsToPush, err := dedupAlerts(alerts)
if err != nil {
return fmt.Errorf("failed to transform alerts for api: %w", err)
return errors.Wrap(err, "failed to transform alerts for api")
}
_, _, err = client.Alerts.Add(ctx, alertsToPush)
if err != nil {
return fmt.Errorf("failed sending alert to LAPI: %w", err)
return errors.Wrap(err, "failed sending alert to LAPI")
}
return nil
}
@ -105,11 +104,11 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
Scenarios: scenarios,
})
if err != nil {
return fmt.Errorf("authenticate watcher (%s): %w", apiConfig.Login, err)
return errors.Wrapf(err, "authenticate watcher (%s)", apiConfig.Login)
}
if err := Client.GetClient().Transport.(*apiclient.JWTTransport).Expiration.UnmarshalText([]byte(authResp.Expire)); err != nil {
return fmt.Errorf("unable to parse jwt expiration: %w", err)
return errors.Wrap(err, "unable to parse jwt expiration")
}
Client.GetClient().Transport.(*apiclient.JWTTransport).Token = authResp.Token

4
cmd/crowdsec/pour.go
View file

@ -12,7 +12,9 @@ import (
)
func runPour(input chan types.Event, holders []leaky.BucketFactory, buckets *leaky.Buckets, cConfig *csconfig.Config) error {
count := 0
var (
count int
)
for {
//bucket is now ready
select {

13
cmd/crowdsec/run_in_svc_windows.go
View file

@ -3,6 +3,7 @@ package main
import (
"fmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"golang.org/x/sys/windows/svc"
@ -21,7 +22,7 @@ func StartRunSvc() error {
isRunninginService, err := svc.IsWindowsService()
if err != nil {
return fmt.Errorf("failed to determine if we are running in windows service mode: %w", err)
return errors.Wrap(err, "failed to determine if we are running in windows service mode")
}
if isRunninginService {
return runService(svcName)
@ -30,22 +31,22 @@ func StartRunSvc() error {
if flags.WinSvc == "Install" {
err = installService(svcName, svcDescription)
if err != nil {
return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err)
return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName)
}
} else if flags.WinSvc == "Remove" {
err = removeService(svcName)
if err != nil {
return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err)
return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName)
}
} else if flags.WinSvc == "Start" {
err = startService(svcName)
if err != nil {
return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err)
return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName)
}
} else if flags.WinSvc == "Stop" {
err = controlService(svcName, svc.Stop, svc.Stopped)
if err != nil {
return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err)
return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName)
}
} else if flags.WinSvc == "" {
return WindowsRun()
@ -65,7 +66,7 @@ func WindowsRun() error {
if err != nil {
return err
}
// Configure logging
log.Infof("Crowdsec %s", version.String())
apiReady := make(chan bool, 1)

35
cmd/crowdsec/serve.go
View file

@ -1,16 +1,16 @@
package main
import (
"fmt"
"os"
"os/signal"
"syscall"
"time"
"github.com/coreos/go-systemd/v22/daemon"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/csdaemon"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
@ -68,7 +68,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
}
apiServer, err := initAPIServer(cConfig)
if err != nil {
return nil, fmt.Errorf("unable to init api server: %w", err)
return nil, errors.Wrap(err, "unable to init api server")
}
apiReady := make(chan bool, 1)
@ -78,7 +78,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
if !cConfig.DisableAgent {
csParsers, err := initCrowdsec(cConfig)
if err != nil {
return nil, fmt.Errorf("unable to init crowdsec: %w", err)
return nil, errors.Wrap(err, "unable to init crowdsec")
}
// restore bucket state
@ -180,13 +180,13 @@ func shutdownCrowdsec() error {
func shutdown(sig os.Signal, cConfig *csconfig.Config) error {
if !cConfig.DisableAgent {
if err := shutdownCrowdsec(); err != nil {
return fmt.Errorf("failed to shut down crowdsec: %w", err)
return errors.Wrap(err, "failed to shut down crowdsec")
}
}
if !cConfig.DisableAPI {
if err := shutdownAPI(); err != nil {
return fmt.Errorf("failed to shut down api routines: %w", err)
return errors.Wrap(err, "failed to shut down api routines")
}
}
@ -238,13 +238,13 @@ func HandleSignals(cConfig *csconfig.Config) error {
log.Warning("SIGHUP received, reloading")
if err = shutdown(s, cConfig); err != nil {
exitChan <- fmt.Errorf("failed shutdown: %w", err)
exitChan <- errors.Wrap(err, "failed shutdown")
break Loop
}
if newConfig, err = reloadHandler(s); err != nil {
exitChan <- fmt.Errorf("reload handler failure: %w", err)
exitChan <- errors.Wrap(err, "reload handler failure")
break Loop
}
@ -256,7 +256,7 @@ func HandleSignals(cConfig *csconfig.Config) error {
case os.Interrupt, syscall.SIGTERM:
log.Warning("SIGTERM received, shutting down")
if err = shutdown(s, cConfig); err != nil {
exitChan <- fmt.Errorf("failed shutdown: %w", err)
exitChan <- errors.Wrap(err, "failed shutdown")
break Loop
}
@ -284,17 +284,17 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
if cConfig.API.Server != nil && cConfig.API.Server.DbConfig != nil {
dbClient, err := database.NewClient(cConfig.API.Server.DbConfig)
if err != nil {
return fmt.Errorf("failed to get database client: %w", err)
return errors.Wrap(err, "failed to get database client")
}
err = exprhelpers.Init(dbClient)
if err != nil {
return fmt.Errorf("failed to init expr helpers: %w", err)
return errors.Wrap(err, "failed to init expr helpers")
}
} else {
err := exprhelpers.Init(nil)
if err != nil {
return fmt.Errorf("failed to init expr helpers: %w", err)
return errors.Wrap(err, "failed to init expr helpers")
}
log.Warningln("Exprhelpers loaded without database client.")
@ -303,7 +303,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
if cConfig.API.CTI != nil && *cConfig.API.CTI.Enabled {
log.Infof("Crowdsec CTI helper enabled")
if err := exprhelpers.InitCrowdsecCTI(cConfig.API.CTI.Key, cConfig.API.CTI.CacheTimeout, cConfig.API.CTI.CacheSize, cConfig.API.CTI.LogLevel); err != nil {
return fmt.Errorf("failed to init crowdsec cti: %w", err)
return errors.Wrap(err, "failed to init crowdsec cti")
}
}
@ -319,7 +319,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
apiServer, err := initAPIServer(cConfig)
if err != nil {
return fmt.Errorf("api server init: %w", err)
return errors.Wrap(err, "api server init")
}
if !flags.TestMode {
@ -332,7 +332,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
if !cConfig.DisableAgent {
csParsers, err := initCrowdsec(cConfig)
if err != nil {
return fmt.Errorf("crowdsec init: %w", err)
return errors.Wrap(err, "crowdsec init")
}
// if it's just linting, we're done
@ -350,7 +350,10 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
}
if cConfig.Common != nil && cConfig.Common.Daemonize {
csdaemon.NotifySystemd(log.StandardLogger())
sent, err := daemon.SdNotify(false, daemon.SdNotifyReady)
if !sent || err != nil {
log.Errorf("Failed to notify(sent: %v): %v", sent, err)
}
// wait for signals
return HandleSignals(cConfig)
}

4
cmd/crowdsec/win_service.go
View file

@ -8,10 +8,10 @@
package main
import (
"fmt"
"syscall"
"time"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"golang.org/x/sys/windows"
"golang.org/x/sys/windows/svc"
@ -106,7 +106,7 @@ func runService(name string) error {
winsvc := crowdsec_winservice{config: cConfig}
if err := svc.Run(name, &winsvc); err != nil {
return fmt.Errorf("%s service failed: %w", name, err)
return errors.Wrapf(err, "%s service failed", name)
}
log.Infof("%s service stopped", name)

4
debian/control vendored
View file

@ -1,8 +1,6 @@
Source: crowdsec
Maintainer: Crowdsec Team <debian@crowdsec.net>
Build-Depends: debhelper, bash
Section: admin
Priority: optional
Build-Depends: debhelper, bash, git
Package: crowdsec
Architecture: any

12
debian/rules vendored
View file

@ -1,6 +1,6 @@
#!/usr/bin/make -f
export DEB_VERSION=$(shell dpkg-parsechangelog | grep -E '^Version:' | cut -f 2 -d ' ')
export DEB_VERSION=$(shell dpkg-parsechangelog | egrep '^Version:' | cut -f 2 -d ' ')
export BUILD_VERSION=v${DEB_VERSION}-debian-pragmatic
export GO111MODULE=on
@ -11,10 +11,12 @@ override_dh_auto_clean:
override_dh_auto_test:
override_dh_auto_build:
override_dh_auto_install:
# just use the prebuilt binaries, otherwise:
# make build BUILD_RE_WASM=0 BUILD_STATIC=1
# mkdir /tmp/go
# echo $(go version)
# echo $($GOCMD version)
# cd cmd/crowdsec && GOROOT=/tmp/go GO111MODULE=on $(GOBUILD) $(LD_OPTS) -o $(CROWDSEC_BIN) -v && cd ..
# cd cmd/crowdsec-cli && GOROOT=/tmp/go GO111MODULE=on $(GOBUILD) $(LD_OPTS) -o cscli -v && cd ..
make build
mkdir -p debian/crowdsec/usr/bin
mkdir -p debian/crowdsec/etc/crowdsec
mkdir -p debian/crowdsec/usr/share/crowdsec

2
debian/templates vendored
View file

@ -17,7 +17,7 @@ Description: Address of the local API server
Template: crowdsec/capi
Type: boolean
Default: true
Description: Do you want to use the centralized remote API server ?
Description: Do you want to the centralized remote API server ?
To share information with other crowdsec you can register to the centralized remote API server.
.
If you don't know what to do, answer yes.

2
docker/docker_start.sh
View file

@ -56,7 +56,7 @@ conf_get() {
if [ $# -ge 2 ]; then
yq e "$1" "$2"
else
cscli config show-yaml | yq e "$1"
yq e "$1" "$CONFIG_FILE"
fi
}

32
docker/test/tests/test_hub_collections.py
View file

@ -6,8 +6,11 @@ Test collection management
from http import HTTPStatus
import json
import os
import pwd
import pytest
import yaml
pytestmark = pytest.mark.docker
@ -82,7 +85,12 @@ def test_taint_bubble_up(crowdsec, tmp_path_factory, flavor):
'COLLECTIONS': f'{coll}'
}
with crowdsec(flavor=flavor, environment=env) as cs:
hub = tmp_path_factory.mktemp("hub")
volumes = {
hub: {'bind': '/etc/crowdsec/hub', 'mode': 'rw'}
}
with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs:
cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
res = cs.cont.exec_run('cscli collections list -o json')
assert res.exit_code == 0
@ -94,13 +102,25 @@ def test_taint_bubble_up(crowdsec, tmp_path_factory, flavor):
f'*Enabled collections : {coll}*',
])
scenario = 'crowdsecurity/http-crawl-non_statics'
# the description won't be read back, it's from the index
yq_command = f"yq -e -i '.description=\"tainted\"' /etc/crowdsec/hub/scenarios/{scenario}.yaml"
res = cs.cont.exec_run(yq_command)
# change file permissions to allow edit
current_uid = pwd.getpwuid(os.getuid()).pw_uid
res = cs.cont.exec_run(f'chown -R {current_uid} /etc/crowdsec/hub')
assert res.exit_code == 0
scenario = 'crowdsecurity/http-crawl-non_statics'
scenario_file = hub / f'scenarios/{scenario}.yaml'
with open(scenario_file) as f:
yml = yaml.safe_load(f)
yml['description'] += ' (tainted)'
# won't be able to read it back because description is taken from the index
with open(scenario_file, 'w') as f:
yaml.dump(yml, f)
with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs:
cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
res = cs.cont.exec_run(f'cscli scenarios inspect {scenario} -o json')
assert res.exit_code == 0
j = json.loads(res.output)

89
go.mod
View file

@ -5,22 +5,14 @@ go 1.20
require (
entgo.io/ent v0.11.3
github.com/AlecAivazis/survey/v2 v2.2.7
github.com/Masterminds/semver/v3 v3.1.1
github.com/Masterminds/sprig/v3 v3.2.2
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/alexliesenfeld/health v0.5.1
github.com/antonmedv/expr v1.12.5
github.com/appleboy/gin-jwt/v2 v2.8.0
github.com/aquasecurity/table v1.8.0
github.com/aws/aws-lambda-go v1.38.0
github.com/aws/aws-sdk-go v1.42.25
github.com/beevik/etree v1.1.0
github.com/blackfireio/osinfo v1.0.3
github.com/bluele/gcache v0.0.2
github.com/buger/jsonparser v1.1.1
github.com/c-robinson/iplib v1.0.3
github.com/cespare/xxhash/v2 v2.1.2
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26
github.com/crowdsecurity/go-cs-lib v0.0.2
github.com/crowdsecurity/grokky v0.2.1
github.com/crowdsecurity/machineid v1.0.2
github.com/davecgh/go-spew v1.1.1
@ -28,7 +20,7 @@ require (
github.com/docker/docker v20.10.24+incompatible
github.com/docker/go-connections v0.4.0
github.com/enescakir/emoji v1.0.0
github.com/fatih/color v1.15.0
github.com/fatih/color v1.13.0
github.com/fsnotify/fsnotify v1.6.0
github.com/gin-gonic/gin v1.7.7
github.com/go-co-op/gocron v1.17.0
@ -37,50 +29,65 @@ require (
github.com/go-openapi/swag v0.19.14
github.com/go-openapi/validate v0.20.0
github.com/go-sql-driver/mysql v1.6.0
github.com/goccy/go-yaml v1.9.7
github.com/gofrs/uuid v4.0.0+incompatible
github.com/golang-jwt/jwt/v4 v4.2.0
github.com/google/go-querystring v1.0.0
github.com/google/uuid v1.3.0
github.com/google/winops v0.0.0-20211216095627-f0e86eb1453b
github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e
github.com/hashicorp/go-hclog v1.0.0
github.com/hashicorp/go-plugin v1.4.2
github.com/hashicorp/go-version v1.2.1
github.com/ivanpirog/coloredcobra v1.0.1
github.com/jackc/pgx/v4 v4.14.1
github.com/jarcoal/httpmock v1.1.0
github.com/jszwec/csvutil v1.5.1
github.com/lithammer/dedent v1.1.0
github.com/mattn/go-isatty v0.0.17
github.com/mattn/go-sqlite3 v1.14.16
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826
github.com/nxadm/tail v1.4.8
github.com/oschwald/geoip2-golang v1.4.0
github.com/oschwald/maxminddb-golang v1.8.0
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.14.0
github.com/prometheus/client_model v0.3.0
github.com/prometheus/prom2json v1.3.0
github.com/r3labs/diff/v2 v2.14.1
github.com/segmentio/kafka-go v0.4.34
github.com/shirou/gopsutil/v3 v3.23.5
github.com/sirupsen/logrus v1.9.2
github.com/spf13/cobra v1.7.0
github.com/stretchr/testify v1.8.3
github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c
github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
github.com/wasilibs/go-re2 v0.2.1
golang.org/x/crypto v0.1.0
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
golang.org/x/mod v0.11.0
golang.org/x/sys v0.9.0
golang.org/x/mod v0.8.0
google.golang.org/grpc v1.47.0
google.golang.org/protobuf v1.28.1
gopkg.in/natefinch/lumberjack.v2 v2.2.1
gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637
gopkg.in/yaml.v2 v2.4.0
gotest.tools/v3 v3.0.3
)
require (
github.com/Masterminds/semver v1.5.0
github.com/Masterminds/sprig/v3 v3.2.2
github.com/aquasecurity/table v1.8.0
github.com/aws/aws-lambda-go v1.38.0
github.com/beevik/etree v1.1.0
github.com/blackfireio/osinfo v1.0.3
github.com/bluele/gcache v0.0.2
github.com/cespare/xxhash/v2 v2.1.2
github.com/corazawaf/coraza/v3 v3.0.0-00010101000000-000000000000
github.com/coreos/go-systemd/v22 v22.5.0
github.com/crowdsecurity/go-cs-lib v0.0.0-20230531105801-4c1535c2b3bd
github.com/goccy/go-yaml v1.9.7
github.com/gofrs/uuid v4.0.0+incompatible
github.com/golang-jwt/jwt/v4 v4.2.0
github.com/google/winops v0.0.0-20211216095627-f0e86eb1453b
github.com/ivanpirog/coloredcobra v1.0.1
github.com/lithammer/dedent v1.1.0
github.com/mattn/go-isatty v0.0.14
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
github.com/segmentio/kafka-go v0.4.34
github.com/shirou/gopsutil/v3 v3.22.12
github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c
github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
github.com/wasilibs/go-re2 v0.2.1
golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc
golang.org/x/sys v0.7.0
gopkg.in/yaml.v3 v3.0.1
k8s.io/apiserver v0.22.5
)
@ -88,7 +95,7 @@ require (
require (
ariga.io/atlas v0.7.2-0.20220927111110-867ee0cca56a // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/Masterminds/semver/v3 v3.1.1 // indirect
github.com/PuerkitoBio/purell v1.1.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/agext/levenshtein v1.2.1 // indirect
@ -96,7 +103,7 @@ require (
github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/corazawaf/libinjection-go v0.1.2 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/docker/distribution v2.8.2+incompatible // indirect
github.com/docker/go-units v0.4.0 // indirect
@ -140,7 +147,7 @@ require (
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/magefile/mage v1.14.0 // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
@ -155,6 +162,7 @@ require (
github.com/oklog/run v1.0.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 // indirect
github.com/pierrec/lz4/v4 v4.1.15 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
@ -163,24 +171,24 @@ require (
github.com/rivo/uniseg v0.2.0 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/shopspring/decimal v1.2.0 // indirect
github.com/spf13/cast v1.3.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/tetratelabs/wazero v1.0.0-rc.2 // indirect
github.com/tidwall/gjson v1.13.0 // indirect
github.com/tidwall/gjson v1.14.4 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
github.com/tklauser/go-sysconf v0.3.11 // indirect
github.com/tklauser/numcpus v0.6.0 // indirect
github.com/ugorji/go/codec v1.2.6 // indirect
github.com/vmihailenco/msgpack v4.0 .4+incompatible // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
github.com/zclconf/go-cty v1.8.0 // indirect
go.mongodb.org/mongo-driver v1.9.4 // indirect
golang.org/x/net v0.7.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/term v0.5.0 // indirect
golang.org/x/text v0.7.0 // indirect
golang.org/x/tools v0.6.0 // indirect
go.mongodb.org/mongo-driver v1.9.0 // indirect
golang.org/x/net v0.9.0 // indirect
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
golang.org/x/term v0.7.0 // indirect
golang.org/x/text v0.9.0 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
@ -190,8 +198,11 @@ require (
k8s.io/apimachinery v0.25.2 // indirect
k8s.io/klog/v2 v2.70.1 // indirect
k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect
rsc.io/binaryregexp v0.2.0 // indirect
sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
)
replace golang.org/x/time/rate => github.com/crowdsecurity/crowdsec/pkg/time/rate v0.0.0
replace github.com/corazawaf/coraza/v3 => ./coraza

85
go.sum
View file

@ -55,12 +55,14 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8=
github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk=
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8 h1:xzYJEypr/85nBpB11F9br+3HUrpgb+fcm5iADzXXYEw=
@ -149,6 +151,8 @@ github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMe
github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
github.com/corazawaf/libinjection-go v0.1.2 h1:oeiV9pc5rvJ+2oqOqXEAMJousPpGiup6f7Y3nZj5GoM=
github.com/corazawaf/libinjection-go v0.1.2/go.mod h1:OP4TM7xdJ2skyXqNX1AN1wN5nNZEmJNuWbNPOItn7aw=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
@ -170,8 +174,8 @@ github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
github.com/crowdsecurity/go-cs-lib v0.0.2 h1:+Tjmf/IclOXNzU9sxKVQvUl9CkMfbM60xQ0zA05NWps=
github.com/crowdsecurity/go-cs-lib v0.0.2/go.mod h1:iznTJ19qLTYdZBcRb5RVDlcUdSlayBCivBkWsXlOY3g=
github.com/crowdsecurity/go-cs-lib v0.0.0-20230531105801-4c1535c2b3bd h1:Y70ceDKAKYFXTnxEjXuBDSh07umvDhbX3PCCYhdtsZ0=
github.com/crowdsecurity/go-cs-lib v0.0.0-20230531105801-4c1535c2b3bd/go.mod h1:9JJLSpGj1ZXnROV3xAcJvS/HTaUvuA8K3gGOpO4tfVc=
github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4=
github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc=
@ -209,12 +213,12 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
@ -657,18 +661,16 @@ github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
@ -681,6 +683,7 @@ github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyex
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI=
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
@ -755,6 +758,8 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8=
github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
@ -831,12 +836,8 @@ github.com/segmentio/kafka-go v0.4.34 h1:Dm6YlLMiVSiwwav20KY0AoY63s661FXevwJ3CVH
github.com/segmentio/kafka-go v0.4.34/go.mod h1:GAjxBQJdQMB5zfNA21AhpaqOB2Mu+w3De4ni3Gbm8y0=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shirou/gopsutil/v3 v3.23.5 h1:5SgDCeQ0KW0S4N0znjeM/eFHXXOKyv2dVNgRq/c9P6Y=
github.com/shirou/gopsutil/v3 v3.23.5/go.mod h1:Ng3Maa27Q2KARVJ0SPZF5NdrQSC3XHKP8IIWrHgMeLY=
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
github.com/shirou/gopsutil/v3 v3.22.12 h1:oG0ns6poeUSxf78JtOsfygNWuEHYYz8hnnNg7P04TJs=
github.com/shirou/gopsutil/v3 v3.22.12/go.mod h1:Xd7P1kwZcp5VW52+9XsirIKd/BROzbb2wdX3Kqlz9uI=
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
@ -887,6 +888,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
@ -895,13 +897,14 @@ github.com/tetratelabs/wazero v1.0.0-rc.2/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+Gk
github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c h1:HelZ2kAFadG0La9d+4htN4HzQ68Bm2iM9qKMSMES6xg=
github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c/go.mod h1:JlzghshsemAMDGZLytTFY8C1JQxQPhnatWqNwUXjggo=
github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/gjson v1.13.0 h1:3TFY9yxOQShrvmjdM76K+jc66zJeT6D3/VFFYCGQf7M=
github.com/tidwall/gjson v1.13.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM=
github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms=
@ -940,8 +943,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zclconf/go-cty v1.8.0 h1:s4AvqaeQzJIu3ndv4gVIhplVD0krU+bgrcLSVUnaWuA=
github.com/zclconf/go-cty v1.8.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk=
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
@ -960,8 +963,8 @@ go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS
go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE=
go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
go.mongodb.org/mongo-driver v1.9.4 h1:qXWlnK2WCOWSxJ/Hm3XyYOGKv3ujA2btBsCyuIFvQjc=
go.mongodb.org/mongo-driver v1.9.4/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY=
go.mongodb.org/mongo-driver v1.9.0 h1:f3aLGJvQmBl8d9S40IL+jEyBC6hfLPbJjv9t5hEM9ck=
go.mongodb.org/mongo-driver v1.9.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@ -1028,8 +1031,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU=
golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -1052,8 +1055,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -1104,8 +1107,8 @@ golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -1125,8 +1128,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -1198,23 +1201,23 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -1224,8 +1227,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@ -1294,7 +1297,6 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -1465,6 +1467,7 @@ k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2R
k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4=
k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

969
mk/__gmsl
View file

@ -1,969 +0,0 @@
# ----------------------------------------------------------------------------
#
# GNU Make Standard Library (GMSL)
#
# A library of functions to be used with GNU Make's $(call) that
# provides functionality not available in standard GNU Make.
#
# Copyright (c) 2005-2022 John Graham-Cumming
#
# This file is part of GMSL
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# Neither the name of the John Graham-Cumming nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# ----------------------------------------------------------------------------
# This is the GNU Make Standard Library version number as a list with
# three items: major, minor, revision
gmsl_version := 1 2 0
__gmsl_name := GNU Make Standard Library
# Used to output warnings and error from the library, it's possible to
# disable any warnings or errors by overriding these definitions
# manually or by setting GMSL_NO_WARNINGS or GMSL_NO_ERRORS
ifdef GMSL_NO_WARNINGS
__gmsl_warning :=
else
__gmsl_warning = $(if $1,$(warning $(__gmsl_name): $1))
endif
ifdef GMSL_NO_ERRORS
__gmsl_error :=
else
__gmsl_error = $(if $1,$(error $(__gmsl_name): $1))
endif
# If GMSL_TRACE is enabled then calls to the library functions are
# traced to stdout using warning messages with their arguments
ifdef GMSL_TRACE
__gmsl_tr1 = $(warning $0('$1'))
__gmsl_tr2 = $(warning $0('$1','$2'))
__gmsl_tr3 = $(warning $0('$1','$2','$3'))
else
__gmsl_tr1 :=
__gmsl_tr2 :=
__gmsl_tr3 :=
endif
# See if spaces are valid in variable names (this was the case until
# GNU Make 3.82)
ifeq ($(MAKE_VERSION),3.82)
__gmsl_spaced_vars := $(false)
else
__gmsl_spaced_vars := $(true)
endif
# Figure out whether we have $(eval) or not (GNU Make 3.80 and above)
# if we do not then output a warning message, if we do then some
# functions will be enabled.
__gmsl_have_eval := $(false)
__gmsl_ignore := $(eval __gmsl_have_eval := $(true))
# If this is being run with Electric Cloud's emake then warn that
# their $(eval) support is incomplete in 1.x, 2.x, 3.x, 4.x and 5.0,
# 5.1, 5.2 and 5.3
ifdef ECLOUD_BUILD_ID
__gmsl_emake_major := $(word 1,$(subst ., ,$(EMAKE_VERSION)))
__gmsl_emake_minor := $(word 2,$(subst ., ,$(EMAKE_VERSION)))
ifneq ("$(findstring $(__gmsl_emake_major),1 2 3 4)$(findstring $(__gmsl_emake_major)$(__gmsl_emake_minor),50 51 52 53)","")
$(warning You are using a version of Electric Cloud's emake which has incomplete $$(eval) support)
__gmsl_have_eval := $(false)
endif
endif
# See if we have $(lastword) (GNU Make 3.81 and above)
__gmsl_have_lastword := $(lastword $(false) $(true))
# See if we have native or and and (GNU Make 3.81 and above)
__or_tt := /$(or $(true),$(true))/$(or $(true),$(false))/$(or $(false),$(true))/$(or $(false),$(false))/
__and_tt := /$(and $(true),$(true))/$(and $(true),$(false))/$(and $(false),$(true))/$(and $(false),$(false))/
__gmsl_have_or := $(if $(filter /T/T/T//,$(__or_tt)),$(true),$(false))
__gmsl_have_and := $(if $(filter /T////,$(__and_tt)),$(true),$(false))
ifneq ($(__gmsl_have_eval),$(true))
$(call __gmsl_warning,Your make version $(MAKE_VERSION) does not support $$$$(eval): some functions disabled)
endif
__gmsl_dollar := $$
__gmsl_hash := \#
# ----------------------------------------------------------------------------
# ----------------------------------------------------------------------------
# Function: gmsl_compatible
# Arguments: List containing the desired library version number (maj min rev)
# Returns: $(true) if this version of the library is compatible
# with the requested version number, otherwise $(false)
# ----------------------------------------------------------------------------
gmsl_compatible = $(strip \
$(if $(call gt,$(word 1,$1),$(word 1,$(gmsl_version))), \
$(false), \
$(if $(call lt,$(word 1,$1),$(word 1,$(gmsl_version))), \
$(true), \
$(if $(call gt,$(word 2,$1),$(word 2,$(gmsl_version))), \
$(false), \
$(if $(call lt,$(word 2,$1),$(word 2,$(gmsl_version))), \
$(true), \
$(call lte,$(word 3,$1),$(word 3,$(gmsl_version))))))))
# ###########################################################################
# LOGICAL OPERATORS
# ###########################################################################
# not is defined in gmsl
# ----------------------------------------------------------------------------
# Function: and
# Arguments: Two boolean values
# Returns: Returns $(true) if both of the booleans are true
# ----------------------------------------------------------------------------
ifneq ($(__gmsl_have_and),$(true))
and = $(__gmsl_tr2)$(if $1,$(if $2,$(true),$(false)),$(false))
endif
# ----------------------------------------------------------------------------
# Function: or
# Arguments: Two boolean values
# Returns: Returns $(true) if either of the booleans is true
# ----------------------------------------------------------------------------
ifneq ($(__gmsl_have_or),$(true))
or = $(__gmsl_tr2)$(if $1$2,$(true),$(false))
endif
# ----------------------------------------------------------------------------
# Function: xor
# Arguments: Two boolean values
# Returns: Returns $(true) if exactly one of the booleans is true
# ----------------------------------------------------------------------------
xor = $(__gmsl_tr2)$(if $1,$(if $2,$(false),$(true)),$(if $2,$(true),$(false)))
# ----------------------------------------------------------------------------
# Function: nand
# Arguments: Two boolean values
# Returns: Returns value of 'not and'
# ----------------------------------------------------------------------------
nand = $(__gmsl_tr2)$(if $1,$(if $2,$(false),$(true)),$(true))
# ----------------------------------------------------------------------------
# Function: nor
# Arguments: Two boolean values
# Returns: Returns value of 'not or'
# ----------------------------------------------------------------------------
nor = $(__gmsl_tr2)$(if $1$2,$(false),$(true))
# ----------------------------------------------------------------------------
# Function: xnor
# Arguments: Two boolean values
# Returns: Returns value of 'not xor'
# ----------------------------------------------------------------------------
xnor =$(__gmsl_tr2)$(if $1,$(if $2,$(true),$(false)),$(if $2,$(false),$(true)))
# ###########################################################################
# LIST MANIPULATION FUNCTIONS
# ###########################################################################
# ----------------------------------------------------------------------------
# Function: first (same as LISP's car, or head)
# Arguments: 1: A list
# Returns: Returns the first element of a list
# ----------------------------------------------------------------------------
first = $(__gmsl_tr1)$(firstword $1)
# ----------------------------------------------------------------------------
# Function: last
# Arguments: 1: A list
# Returns: Returns the last element of a list
# ----------------------------------------------------------------------------
ifeq ($(__gmsl_have_lastword),$(true))
last = $(__gmsl_tr1)$(lastword $1)
else
last = $(__gmsl_tr1)$(if $1,$(word $(words $1),$1))
endif
# ----------------------------------------------------------------------------
# Function: rest (same as LISP's cdr, or tail)
# Arguments: 1: A list
# Returns: Returns the list with the first element removed
# ----------------------------------------------------------------------------
rest = $(__gmsl_tr1)$(wordlist 2,$(words $1),$1)
# ----------------------------------------------------------------------------
# Function: chop
# Arguments: 1: A list
# Returns: Returns the list with the last element removed
# ----------------------------------------------------------------------------
chop = $(__gmsl_tr1)$(wordlist 2,$(words $1),x $1)
# ----------------------------------------------------------------------------
# Function: map
# Arguments: 1: Name of function to $(call) for each element of list
# 2: List to iterate over calling the function in 1
# Returns: The list after calling the function on each element
# ----------------------------------------------------------------------------
map = $(__gmsl_tr2)$(strip $(foreach a,$2,$(call $1,$a)))
# ----------------------------------------------------------------------------
# Function: pairmap
# Arguments: 1: Name of function to $(call) for each pair of elements
# 2: List to iterate over calling the function in 1
# 3: Second list to iterate over calling the function in 1
# Returns: The list after calling the function on each pair of elements
# ----------------------------------------------------------------------------
pairmap = $(strip $(__gmsl_tr3)\
$(if $2$3,$(call $1,$(call first,$2),$(call first,$3)) \
$(call pairmap,$1,$(call rest,$2),$(call rest,$3))))
# ----------------------------------------------------------------------------
# Function: leq
# Arguments: 1: A list to compare against...
# 2: ...this list
# Returns: Returns $(true) if the two lists are identical
# ----------------------------------------------------------------------------
leq = $(__gmsl_tr2)$(strip $(if $(call seq,$(words $1),$(words $2)), \
$(call __gmsl_list_equal,$1,$2),$(false)))
__gmsl_list_equal = $(if $(strip $1), \
$(if $(call seq,$(call first,$1),$(call first,$2)), \
$(call __gmsl_list_equal, \
$(call rest,$1), \
$(call rest,$2)), \
$(false)), \
$(true))
# ----------------------------------------------------------------------------
# Function: lne
# Arguments: 1: A list to compare against...
# 2: ...this list
# Returns: Returns $(true) if the two lists are different
# ----------------------------------------------------------------------------
lne = $(__gmsl_tr2)$(call not,$(call leq,$1,$2))
# ----------------------------------------------------------------------------
# Function: reverse
# Arguments: 1: A list to reverse
# Returns: The list with its elements in reverse order
# ----------------------------------------------------------------------------
reverse =$(__gmsl_tr1)$(strip $(if $1,$(call reverse,$(call rest,$1)) \
$(call first,$1)))
# ----------------------------------------------------------------------------
# Function: uniq
# Arguments: 1: A list from which to remove repeated elements
# Returns: The list with duplicate elements removed without reordering
# ----------------------------------------------------------------------------
uniq = $(strip $(__gmsl_tr1) $(if $1,$(firstword $1) \
$(call uniq,$(filter-out $(firstword $1),$1))))
# ----------------------------------------------------------------------------
# Function: length
# Arguments: 1: A list
# Returns: The number of elements in the list
# ----------------------------------------------------------------------------
length = $(__gmsl_tr1)$(words $1)
# ###########################################################################
# STRING MANIPULATION FUNCTIONS
# ###########################################################################
# Helper function that translates any GNU Make 'true' value (i.e. a
# non-empty string) to our $(true)
__gmsl_make_bool = $(if $(strip $1),$(true),$(false))
# ----------------------------------------------------------------------------
# Function: seq
# Arguments: 1: A string to compare against...
# 2: ...this string
# Returns: Returns $(true) if the two strings are identical
# ----------------------------------------------------------------------------
seq = $(__gmsl_tr2)$(if $(subst x$1,,x$2)$(subst x$2,,x$1),$(false),$(true))
# ----------------------------------------------------------------------------
# Function: sne
# Arguments: 1: A string to compare against...
# 2: ...this string
# Returns: Returns $(true) if the two strings are not the same
# ----------------------------------------------------------------------------
sne = $(__gmsl_tr2)$(call not,$(call seq,$1,$2))
# ----------------------------------------------------------------------------
# Function: split
# Arguments: 1: The character to split on
# 2: A string to split
# Returns: Splits a string into a list separated by spaces at the split
# character in the first argument
# ----------------------------------------------------------------------------
split = $(__gmsl_tr2)$(strip $(subst $1, ,$2))
# ----------------------------------------------------------------------------
# Function: merge
# Arguments: 1: The character to put between fields
# 2: A list to merge into a string
# Returns: Merges a list into a single string, list elements are separated
# by the character in the first argument
# ----------------------------------------------------------------------------
merge = $(__gmsl_tr2)$(strip $(if $2, \
$(if $(call seq,1,$(words $2)), \
$2,$(call first,$2)$1$(call merge,$1,$(call rest,$2)))))
ifdef __gmsl_have_eval
# ----------------------------------------------------------------------------
# Function: tr
# Arguments: 1: The list of characters to translate from
# 2: The list of characters to translate to
# 3: The text to translate
# Returns: Returns the text after translating characters
# ----------------------------------------------------------------------------
tr = $(strip $(__gmsl_tr3)$(call assert_no_dollar,$0,$1$2$3) \
$(eval __gmsl_t := $3) \
$(foreach c, \
$(join $(addsuffix :,$1),$2), \
$(eval __gmsl_t := \
$(subst $(word 1,$(subst :, ,$c)),$(word 2,$(subst :, ,$c)), \
$(__gmsl_t))))$(__gmsl_t))
# Common character classes for use with the tr function. Each of
# these is actually a variable declaration and must be wrapped with
# $() or ${} to be used.
[A-Z] := A B C D E F G H I J K L M N O P Q R S T U V W X Y Z #
[a-z] := a b c d e f g h i j k l m n o p q r s t u v w x y z #
[0-9] := 0 1 2 3 4 5 6 7 8 9 #
[A-F] := A B C D E F #
# ----------------------------------------------------------------------------
# Function: uc
# Arguments: 1: Text to upper case
# Returns: Returns the text in upper case
# ----------------------------------------------------------------------------
uc = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(call tr,$([a-z]),$([A-Z]),$1)
# ----------------------------------------------------------------------------
# Function: lc
# Arguments: 1: Text to lower case
# Returns: Returns the text in lower case
# ----------------------------------------------------------------------------
lc = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(call tr,$([A-Z]),$([a-z]),$1)
# ----------------------------------------------------------------------------
# Function: strlen
# Arguments: 1: A string
# Returns: Returns the length of the string
# ----------------------------------------------------------------------------
# This results in __gmsl_tab containing a tab
__gmsl_tab := #
__gmsl_characters := A B C D E F G H I J K L M N O P Q R S T U V W X Y Z
__gmsl_characters += a b c d e f g h i j k l m n o p q r s t u v w x y z
__gmsl_characters += 0 1 2 3 4 5 6 7 8 9
__gmsl_characters += ` ~ ! @ \# $$ % ^ & * ( ) - _ = +
__gmsl_characters += { } [ ] \ : ; ' " < > , . / ? |
# This results in __gmsl_space containing just a space
__gmsl_empty :=
__gmsl_space := $(__gmsl_empty) $(__gmsl_empty)
strlen = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(strip $(eval __temp := $(subst $(__gmsl_space),x,$1))$(foreach a,$(__gmsl_characters),$(eval __temp := $$(subst $$a,x,$(__temp))))$(eval __temp := $(subst x,x ,$(__temp)))$(words $(__temp)))
# This results in __gmsl_newline containing just a newline
define __gmsl_newline
endef
# ----------------------------------------------------------------------------
# Function: substr
# Arguments: 1: A string
# 2: Start position (first character is 1)
# 3: End position (inclusive)
# Returns: A substring.
# Note: The string in $1 must not contain a §
# ----------------------------------------------------------------------------
substr = $(if $2,$(__gmsl_tr3)$(call assert_no_dollar,$0,$1$2$3)$(strip $(eval __temp := $$(subst $$(__gmsl_space),§ ,$$1))$(foreach a,$(__gmsl_characters),$(eval __temp := $$(subst $$a,$$a$$(__gmsl_space),$(__temp))))$(eval __temp := $(wordlist $2,$3,$(__temp))))$(subst §,$(__gmsl_space),$(subst $(__gmsl_space),,$(__temp))))
endif # __gmsl_have_eval
# ###########################################################################
# SET MANIPULATION FUNCTIONS
# ###########################################################################
# Sets are represented by sorted, deduplicated lists. To create a set
# from a list use set_create, or start with the empty_set and
# set_insert individual elements
# This is the empty set
empty_set :=
# ----------------------------------------------------------------------------
# Function: set_create
# Arguments: 1: A list of set elements
# Returns: Returns the newly created set
# ----------------------------------------------------------------------------
set_create = $(__gmsl_tr1)$(sort $1)
# ----------------------------------------------------------------------------
# Function: set_insert
# Arguments: 1: A single element to add to a set
# 2: A set
# Returns: Returns the set with the element added
# ----------------------------------------------------------------------------
set_insert = $(__gmsl_tr2)$(sort $1 $2)
# ----------------------------------------------------------------------------
# Function: set_remove
# Arguments: 1: A single element to remove from a set
# 2: A set
# Returns: Returns the set with the element removed
# ----------------------------------------------------------------------------
set_remove = $(__gmsl_tr2)$(filter-out $1,$2)
# ----------------------------------------------------------------------------
# Function: set_is_member, set_is_not_member
# Arguments: 1: A single element
# 2: A set
# Returns: (set_is_member) Returns $(true) if the element is in the set
# (set_is_not_member) Returns $(false) if the element is in the set
# ----------------------------------------------------------------------------
set_is_member = $(__gmsl_tr2)$(if $(filter $1,$2),$(true),$(false))
set_is_not_member = $(__gmsl_tr2)$(if $(filter $1,$2),$(false),$(true))
# ----------------------------------------------------------------------------
# Function: set_union
# Arguments: 1: A set
# 2: Another set
# Returns: Returns the union of the two sets
# ----------------------------------------------------------------------------
set_union = $(__gmsl_tr2)$(sort $1 $2)
# ----------------------------------------------------------------------------
# Function: set_intersection
# Arguments: 1: A set
# 2: Another set
# Returns: Returns the intersection of the two sets
# ----------------------------------------------------------------------------
set_intersection = $(__gmsl_tr2)$(filter $1,$2)
# ----------------------------------------------------------------------------
# Function: set_is_subset
# Arguments: 1: A set
# 2: Another set
# Returns: Returns $(true) if the first set is a subset of the second
# ----------------------------------------------------------------------------
set_is_subset = $(__gmsl_tr2)$(call set_equal,$(call set_intersection,$1,$2),$1)
# ----------------------------------------------------------------------------
# Function: set_equal
# Arguments: 1: A set
# 2: Another set
# Returns: Returns $(true) if the two sets are identical
# ----------------------------------------------------------------------------
set_equal = $(__gmsl_tr2)$(call seq,$1,$2)
# ###########################################################################
# ARITHMETIC LIBRARY
# ###########################################################################
# Integers a represented by lists with the equivalent number of x's.
# For example the number 4 is x x x x.
# ----------------------------------------------------------------------------
# Function: int_decode
# Arguments: 1: A number of x's representation
# Returns: Returns the integer for human consumption that is represented
# by the string of x's
# ----------------------------------------------------------------------------
int_decode = $(__gmsl_tr1)$(if $1,$(if $(call seq,$(word 1,$1),x),$(words $1),$1),0)
# ----------------------------------------------------------------------------
# Function: int_encode
# Arguments: 1: A number in human-readable integer form
# Returns: Returns the integer encoded as a string of x's
# ----------------------------------------------------------------------------
__int_encode = $(if $1,$(if $(call seq,$(words $(wordlist 1,$1,$2)),$1),$(wordlist 1,$1,$2),$(call __int_encode,$1,$(if $2,$2 $2,x))))
__strip_leading_zero = $(if $1,$(if $(call seq,$(patsubst 0%,%,$1),$1),$1,$(call __strip_leading_zero,$(patsubst 0%,%,$1))),0)
int_encode = $(__gmsl_tr1)$(call __int_encode,$(call __strip_leading_zero,$1))
# The arithmetic library functions come in two forms: one form of each
# function takes integers as arguments and the other form takes the
# encoded form (x's created by a call to int_encode). For example,
# there are two plus functions:
#
# plus Called with integer arguments and returns an integer
# int_plus Called with encoded arguments and returns an encoded result
#
# plus will be slower than int_plus because its arguments and result
# have to be translated between the x's format and integers. If doing
# a complex calculation use the int_* forms with a single encoding of
# inputs and single decoding of the output. For simple calculations
# the direct forms can be used.
# Helper function used to wrap an int_* function into a function that
# takes a pair of integers, perhaps a function and returns an integer
# result
__gmsl_int_wrap = $(call int_decode,$(call $1,$(call int_encode,$2),$(call int_encode,$3)))
__gmsl_int_wrap1 = $(call int_decode,$(call $1,$(call int_encode,$2)))
__gmsl_int_wrap2 = $(call $1,$(call int_encode,$2),$(call int_encode,$3))
# ----------------------------------------------------------------------------
# Function: int_plus
# Arguments: 1: A number in x's representation
# 2: Another number in x's represntation
# Returns: Returns the sum of the two numbers in x's representation
# ----------------------------------------------------------------------------
int_plus = $(strip $(__gmsl_tr2)$1 $2)
# ----------------------------------------------------------------------------
# Function: plus (wrapped version of int_plus)
# Arguments: 1: An integer
# 2: Another integer
# Returns: Returns the sum of the two integers
# ----------------------------------------------------------------------------
plus = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_plus,$1,$2)
# ----------------------------------------------------------------------------
# Function: int_subtract
# Arguments: 1: A number in x's representation
# 2: Another number in x's represntation
# Returns: Returns the difference of the two numbers in x's representation,
# or outputs an error on a numeric underflow
# ----------------------------------------------------------------------------
int_subtract = $(strip $(__gmsl_tr2)$(if $(call int_gte,$1,$2), \
$(filter-out xx,$(join $1,$2)), \
$(call __gmsl_warning,Subtraction underflow)))
# ----------------------------------------------------------------------------
# Function: subtract (wrapped version of int_subtract)
# Arguments: 1: An integer
# 2: Another integer
# Returns: Returns the difference of the two integers,
# or outputs an error on a numeric underflow
# ----------------------------------------------------------------------------
subtract = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_subtract,$1,$2)
# ----------------------------------------------------------------------------
# Function: int_multiply
# Arguments: 1: A number in x's representation
# 2: Another number in x's represntation
# Returns: Returns the product of the two numbers in x's representation
# ----------------------------------------------------------------------------
int_multiply = $(strip $(__gmsl_tr2)$(foreach a,$1,$2))
# ----------------------------------------------------------------------------
# Function: multiply (wrapped version of int_multiply)
# Arguments: 1: An integer
# 2: Another integer
# Returns: Returns the product of the two integers
# ----------------------------------------------------------------------------
multiply = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_multiply,$1,$2)
# ----------------------------------------------------------------------------
# Function: int_divide
# Arguments: 1: A number in x's representation
# 2: Another number in x's represntation
# Returns: Returns the result of integer division of argument 1 divided
# by argument 2 in x's representation
# ----------------------------------------------------------------------------
int_divide = $(__gmsl_tr2)$(strip $(if $1,$(if $2, \
$(subst M,x,$(filter-out x,$(subst $2,M,$1))), \
$(call __gmsl_error,Division by zero))))
# ----------------------------------------------------------------------------
# Function: divide (wrapped version of int_divide)
# Arguments: 1: An integer
# 2: Another integer
# Returns: Returns the integer division of the first argument by the second
# ----------------------------------------------------------------------------
divide = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_divide,$1,$2)
# ----------------------------------------------------------------------------
# Function: int_modulo
# Arguments: 1: A number in x's representation
# 2: Another number in x's represntation
# Returns: Returns the remainder of integer division of argument 1 divided
# by argument 2 in x's representation
# ----------------------------------------------------------------------------
int_modulo = $(__gmsl_tr2)$(strip $(if $1,$(if $2, \
$(filter-out M,$(subst $2,M,$1)), \
$(call __gmsl_error,Division by zero))))
# ----------------------------------------------------------------------------
# Function: modulo (wrapped version of int_modulo)
# Arguments: 1: An integer
# 2: Another integer
# Returns: Returns the remainder of integer division of the first argument
# by the second
# ----------------------------------------------------------------------------
modulo = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_modulo,$1,$2)
# ----------------------------------------------------------------------------
# Function: int_max, int_min
# Arguments: 1: A number in x's representation
# 2: Another number in x's represntation
# Returns: Returns the maximum or minimum of its arguments in x's
# representation
# ----------------------------------------------------------------------------
int_max = $(__gmsl_tr2)$(subst xx,x,$(join $1,$2))
int_min = $(__gmsl_tr2)$(subst xx,x,$(filter xx,$(join $1,$2)))
# ----------------------------------------------------------------------------
# Function: max, min
# Arguments: 1: An integer
# 2: Another integer
# Returns: Returns the maximum or minimum of its integer arguments
# ----------------------------------------------------------------------------
max = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_max,$1,$2)
min = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_min,$1,$2)
# ----------------------------------------------------------------------------
# Function: int_gt, int_gte, int_lt, int_lte, int_eq, int_ne
# Arguments: Two x's representation numbers to be compared
# Returns: $(true) or $(false)
#
# int_gt First argument greater than second argument
# int_gte First argument greater than or equal to second argument
# int_lt First argument less than second argument
# int_lte First argument less than or equal to second argument
# int_eq First argument is numerically equal to the second argument
# int_ne First argument is not numerically equal to the second argument
# ----------------------------------------------------------------------------
int_gt = $(__gmsl_tr2)$(call __gmsl_make_bool, \
$(filter-out $(words $2), \
$(words $(call int_max,$1,$2))))
int_gte = $(__gmsl_tr2)$(call __gmsl_make_bool, \
$(call int_gt,$1,$2)$(call int_eq,$1,$2))
int_lt = $(__gmsl_tr2)$(call __gmsl_make_bool, \
$(filter-out $(words $1), \
$(words $(call int_max,$1,$2))))
int_lte = $(__gmsl_tr2)$(call __gmsl_make_bool, \
$(call int_lt,$1,$2)$(call int_eq,$1,$2))
int_eq = $(__gmsl_tr2)$(call __gmsl_make_bool, \
$(filter $(words $1),$(words $2)))
int_ne = $(__gmsl_tr2)$(call __gmsl_make_bool, \
$(filter-out $(words $1),$(words $2)))
# ----------------------------------------------------------------------------
# Function: gt, gte, lt, lte, eq, ne
# Arguments: Two integers to be compared
# Returns: $(true) or $(false)
#
# gt First argument greater than second argument
# gte First argument greater than or equal to second argument
# lt First argument less than second argument
# lte First argument less than or equal to second argument
# eq First argument is numerically equal to the second argument
# ne First argument is not numerically equal to the second argument
# ----------------------------------------------------------------------------
gt = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_gt,$1,$2)
gte = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_gte,$1,$2)
lt = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_lt,$1,$2)
lte = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_lte,$1,$2)
eq = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_eq,$1,$2)
ne = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_ne,$1,$2)
# increment adds 1 to its argument, decrement subtracts 1. Note that
# decrement does not range check and hence will not underflow, but
# will incorrectly say that 0 - 1 = 0
# ----------------------------------------------------------------------------
# Function: int_inc
# Arguments: 1: A number in x's representation
# Returns: The number incremented by 1 in x's representation
# ----------------------------------------------------------------------------
int_inc = $(strip $(__gmsl_tr1)$1 x)
# ----------------------------------------------------------------------------
# Function: inc
# Arguments: 1: An integer
# Returns: The argument incremented by 1
# ----------------------------------------------------------------------------
inc = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_inc,$1)
# ----------------------------------------------------------------------------
# Function: int_dec
# Arguments: 1: A number in x's representation
# Returns: The number decremented by 1 in x's representation
# ----------------------------------------------------------------------------
int_dec = $(__gmsl_tr1)$(strip \
$(if $(call sne,0,$(words $1)), \
$(wordlist 2,$(words $1),$1)))
# ----------------------------------------------------------------------------
# Function: dec
# Arguments: 1: An integer
# Returns: The argument decremented by 1
# ----------------------------------------------------------------------------
dec = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_dec,$1)
# double doubles its argument, and halve halves it
# ----------------------------------------------------------------------------
# Function: int_double
# Arguments: 1: A number in x's representation
# Returns: The number doubled (i.e. * 2) and returned in x's representation
# ----------------------------------------------------------------------------
int_double = $(strip $(__gmsl_tr1)$1 $1)
# ----------------------------------------------------------------------------
# Function: double
# Arguments: 1: An integer
# Returns: The integer times 2
# ----------------------------------------------------------------------------
double = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_double,$1)
# ----------------------------------------------------------------------------
# Function: int_halve
# Arguments: 1: A number in x's representation
# Returns: The number halved (i.e. / 2) and returned in x's representation
# ----------------------------------------------------------------------------
int_halve = $(__gmsl_tr1)$(strip $(subst xx,x,$(filter-out xy x y, \
$(join $1,$(foreach a,$1,y x)))))
# ----------------------------------------------------------------------------
# Function: halve
# Arguments: 1: An integer
# Returns: The integer divided by 2
# ----------------------------------------------------------------------------
halve = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_halve,$1)
# ----------------------------------------------------------------------------
# Function: sequence
# Arguments: 1: An integer
# 2: An integer
# Returns: The sequence [arg1, arg2] of integers if arg1 < arg2 or
# [arg2, arg1] if arg2 > arg1. If arg1 == arg1 return [arg1]
# ----------------------------------------------------------------------------
sequence = $(__gmsl_tr2)$(strip $(if $(call lte,$1,$2), \
$(call __gmsl_sequence_up,$1,$2), \
$(call __gmsl_sequence_dn,$2,$1)))
__gmsl_sequence_up = $(if $(call seq,$1,$2),$1,$1 $(call __gmsl_sequence_up,$(call inc,$1),$2))
__gmsl_sequence_dn = $(if $(call seq,$1,$2),$1,$2 $(call __gmsl_sequence_dn,$1,$(call dec,$2)))
# ----------------------------------------------------------------------------
# Function: dec2hex, dec2bin, dec2oct
# Arguments: 1: An integer
# Returns: The decimal argument converted to hexadecimal, binary or
# octal
# ----------------------------------------------------------------------------
__gmsl_digit = $(subst 15,f,$(subst 14,e,$(subst 13,d,$(subst 12,c,$(subst 11,b,$(subst 10,a,$1))))))
dec2hex = $(call __gmsl_dec2base,$(call int_encode,$1),$(call int_encode,16))
dec2bin = $(call __gmsl_dec2base,$(call int_encode,$1),$(call int_encode,2))
dec2oct = $(call __gmsl_dec2base,$(call int_encode,$1),$(call int_encode,8))
__gmsl_base_divide = $(subst $2,X ,$1)
__gmsl_q = $(strip $(filter X,$1))
__gmsl_r = $(words $(filter x,$1))
__gmsl_dec2base = $(eval __gmsl_temp := $(call __gmsl_base_divide,$1,$2))$(call __gmsl_dec2base_,$(call __gmsl_q,$(__gmsl_temp)),$(call __gmsl_r,$(__gmsl_temp)),$2)
__gmsl_dec2base_ = $(if $1,$(call __gmsl_dec2base,$(subst X,x,$1),$3))$(call __gmsl_digit,$2)
ifdef __gmsl_have_eval
# ###########################################################################
# ASSOCIATIVE ARRAYS
# ###########################################################################
# Magic string that is very unlikely to appear in a key or value
__gmsl_aa_magic := faf192c8efbc25c27992c5bc5add390393d583c6
# ----------------------------------------------------------------------------
# Function: set
# Arguments: 1: Name of associative array
# 2: The key value to associate
# 3: The value associated with the key
# Returns: Nothing
# ----------------------------------------------------------------------------
set = $(__gmsl_tr3)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2$3)$(eval __gmsl_aa_$1_$(__gmsl_aa_magic)_$2_gmsl_aa_$1 := $3)
# Only used internally by memoize function
__gmsl_set = $(call set,$1,$2,$3)$3
# ----------------------------------------------------------------------------
# Function: get
# Arguments: 1: Name of associative array
# 2: The key to retrieve
# Returns: The value stored in the array for that key
# ----------------------------------------------------------------------------
get = $(strip $(__gmsl_tr2)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2)$(__gmsl_aa_$1_$(__gmsl_aa_magic)_$2_gmsl_aa_$1))
# ----------------------------------------------------------------------------
# Function: keys
# Arguments: 1: Name of associative array
# Returns: Returns a list of all defined keys in the array
# ----------------------------------------------------------------------------
keys = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(sort $(patsubst __gmsl_aa_$1_$(__gmsl_aa_magic)_%_gmsl_aa_$1,%, \
$(filter __gmsl_aa_$1_$(__gmsl_aa_magic)_%_gmsl_aa_$1,$(.VARIABLES))))
# ----------------------------------------------------------------------------
# Function: defined
# Arguments: 1: Name of associative array
# 2: The key to test
# Returns: Returns true if the key is defined (i.e. not empty)
# ----------------------------------------------------------------------------
defined = $(__gmsl_tr2)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2)$(call sne,$(call get,$1,$2),)
endif # __gmsl_have_eval
ifdef __gmsl_have_eval
# ###########################################################################
# NAMED STACKS
# ###########################################################################
# ----------------------------------------------------------------------------
# Function: push
# Arguments: 1: Name of stack
# 2: Value to push onto the top of the stack (must not contain
# a space)
# Returns: None
# ----------------------------------------------------------------------------
push = $(__gmsl_tr2)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2)$(eval __gmsl_stack_$1 := $2 $(if $(filter-out undefined,\
$(origin __gmsl_stack_$1)),$(__gmsl_stack_$1)))
# ----------------------------------------------------------------------------
# Function: pop
# Arguments: 1: Name of stack
# Returns: Top element from the stack after removing it
# ----------------------------------------------------------------------------
pop = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(strip $(if $(filter-out undefined,$(origin __gmsl_stack_$1)), \
$(call first,$(__gmsl_stack_$1)) \
$(eval __gmsl_stack_$1 := $(call rest,$(__gmsl_stack_$1)))))
# ----------------------------------------------------------------------------
# Function: peek
# Arguments: 1: Name of stack
# Returns: Top element from the stack without removing it
# ----------------------------------------------------------------------------
peek = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(call first,$(__gmsl_stack_$1))
# ----------------------------------------------------------------------------
# Function: depth
# Arguments: 1: Name of stack
# Returns: Number of items on the stack
# ----------------------------------------------------------------------------
depth = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(words $(__gmsl_stack_$1))
endif # __gmsl_have_eval
ifdef __gmsl_have_eval
# ###########################################################################
# STRING CACHE
# ###########################################################################
# ----------------------------------------------------------------------------
# Function: memoize
# Arguments: 1. Name of the function to be called if the string
# has not been previously seen
# 2. A string
# Returns: Returns the result of a memo function (which the user must
# define) on the passed in string and remembers the result.
#
# Example: Set memo = $(shell echo "$1" | md5sum) to make a cache
# of MD5 hashes of strings. $(call memoize,memo,foo bar baz)
# ----------------------------------------------------------------------------
__gmsl_memoize = $(subst $(__gmsl_space),§,$1)cc2af1bb7c4482f2ba75e338b963d3e7$(subst $(__gmsl_space),§,$2)
memoize = $(__gmsl_tr2)$(strip $(if $(call defined,__gmsl_m,$(__gmsl_memoize)),\
$(call get,__gmsl_m,$(__gmsl_memoize)), \
$(call __gmsl_set,__gmsl_m,$(__gmsl_memoize),$(call $1,$2))))
endif # __gmsl_have_eval
# ###########################################################################
# DEBUGGING FACILITIES
# ###########################################################################
# ----------------------------------------------------------------------------
# Target: gmsl-echo-%
# Arguments: The % should be replaced by the name of a variable that you
# wish to print out.
# Action: Echos the value of the variable that matches the %.
# For example, 'make gmsl-echo-SHELL' will output the value of
# the SHELL variable.
# ----------------------------------------------------------------------------
gmsl-echo-%: ; @echo $($*)
# ----------------------------------------------------------------------------
# Target: gmsl-print-%
# Arguments: The % should be replaced by the name of a variable that you
# wish to print out.
# Action: Echos the name of the variable that matches the % and its value.
# For example, 'make gmsl-print-SHELL' will output the value of
# the SHELL variable
# ----------------------------------------------------------------------------
gmsl-print-%: ; @echo $* = $($*)
# ----------------------------------------------------------------------------
# Function: assert
# Arguments: 1: A boolean that must be true or the assertion will fail
# 2: The message to print with the assertion
# Returns: None
# ----------------------------------------------------------------------------
assert = $(if $2,$(if $1,,$(call __gmsl_error,Assertion failure: $2)))
# ----------------------------------------------------------------------------
# Function: assert_exists
# Arguments: 1: Name of file that must exist, if it is missing an assertion
# will be generated
# Returns: None
# ----------------------------------------------------------------------------
assert_exists = $(if $0,$(call assert,$(wildcard $1),file '$1' missing))
# ----------------------------------------------------------------------------
# Function: assert_no_dollar
# Arguments: 1: Name of a function being executd
# 2: Arguments to check
# Returns: None
# ----------------------------------------------------------------------------
assert_no_dollar = $(call __gmsl_tr2)$(call assert,$(call not,$(findstring $(__gmsl_dollar),$2)),$1 called with a dollar sign in argument)
# ----------------------------------------------------------------------------
# Function: assert_no_space
# Arguments: 1: Name of a function being executd
# 2: Arguments to check
# Returns: None
# ----------------------------------------------------------------------------
ifeq ($(__gmsl_spaced_vars),$(false))
assert_no_space = $(call assert,$(call not,$(findstring $(__gmsl_aa_magic),$(subst $(__gmsl_space),$(__gmsl_aa_magic),$2))),$1 called with a space in argument)
else
assert_no_space =
endif

85
mk/gmsl
View file

@ -1,85 +0,0 @@
# ----------------------------------------------------------------------------
#
# GNU Make Standard Library (GMSL)
#
# A library of functions to be used with GNU Make's $(call) that
# provides functionality not available in standard GNU Make.
#
# Copyright (c) 2005-2022 John Graham-Cumming
#
# This file is part of GMSL
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# Neither the name of the John Graham-Cumming nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# ----------------------------------------------------------------------------
# Determine if the library has already been included and if so don't
# bother including it again
ifndef __gmsl_included
# Standard definitions for true and false. true is any non-empty
# string, false is an empty string. These are intended for use with
# $(if).
true := T
false :=
# ----------------------------------------------------------------------------
# Function: not
# Arguments: 1: A boolean value
# Returns: Returns the opposite of the arg. (true -> false, false -> true)
# ----------------------------------------------------------------------------
not = $(if $1,$(false),$(true))
# Prevent reinclusion of the library
__gmsl_included := $(true)
# Try to determine where this file is located. If the caller did
# include /foo/gmsl then extract the /foo/ so that __gmsl gets
# included transparently
__gmsl_root :=
ifneq ($(MAKEFILE_LIST),)
__gmsl_root := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST))
# If there are any spaces in the path in __gmsl_root then give up
ifeq (1,$(words $(__gmsl_root)))
__gmsl_root := $(patsubst %gmsl,%,$(__gmsl_root))
endif
endif
include $(__gmsl_root)__gmsl
endif # __gmsl_included

733
mk/gmsl.html
View file

@ -1,733 +0,0 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html><head>
<meta content="text/html; charset=ISO-8859-1" http-equiv="content-type">
<title>GNU Make Standard Library</title></head>
<body>
<h1>GNU Make Standard Library</h1>
The GNU Make Standard Library (GMSL) is a collection of functions
implemented using native GNU Make functionality that provide list and
string manipulation, integer arithmetic, associative arrays, stacks,
and debugging facilities.&nbsp; The GMSL is released under the BSD License.<br>
<br>
<a href="https://github.com/jgrahamc/gmsl/">[Project Page]</a>
<a href="https://github.com/jgrahamc/gmsl/releases/">[Releases]</a>
<br>
<h2>Using GMSL</h2>
The two files needed are <span style="font-family: monospace;">gmsl</span>
and <span style="font-family: monospace;">__gmsl</span>.&nbsp; To
include the GMSL in your Makefile do<br>
<pre style="margin-left: 40px;">include gmsl</pre>
<span style="font-family: monospace;">gmsl</span> automatically includes<span style="font-family: monospace;"> __gmsl</span>.&nbsp; To check that
you have the right version of <span style="font-family: monospace;">gmsl</span>
use the <span style="font-family: monospace;">gmsl_compatible</span>
function (see
below). The current version is <span style="font-family: monospace;">1 2 0</span>.<br>
<br>
The GMSL package also includes a test suite for GMSL.&nbsp; Just run <span style="font-family: monospace;">make -f gmsl-tests</span>.<br>
<h2>Logical Operators</h2>GMSL has boolean $(true) (a non-empty string)
and $(false) (an empty string).&nbsp; The following operators can be
used with those variables.<br>
<br>
<hr style="width: 100%; height: 2px;"><span style="font-weight: bold;">not</span><br>
<br>
<span style="font-family: monospace;">Arguments: A boolean value</span><br style="font-family: monospace;">
<span style="font-family: monospace;">Returns:&nbsp;&nbsp; Returns $(true) if the boolean is $(false) and vice versa</span><br style="font-family: monospace;">
<hr style="width: 100%; height: 2px; font-family: monospace;"><span style="font-weight: bold;"></span><span style="font-weight: bold;">and</span><br>
<br>
<span style="font-family: monospace;">Arguments: Two boolean values</span><br style="font-family: monospace;">
<span style="font-family: monospace;">Returns:&nbsp;&nbsp; Returns $(true) if both of the booleans are true</span><br style="font-family: monospace;">
<hr style="width: 100%; height: 2px; font-family: monospace;"><span style="font-weight: bold;">or</span><br>
<br>
<span style="font-family: monospace;">Arguments: Two boolean values</span><br style="font-family: monospace;">
<span style="font-family: monospace;">Returns:&nbsp;&nbsp; Returns $(true) if either of the booleans is true</span><br style="font-family: monospace;">
<hr style="width: 100%; height: 2px; font-family: monospace;"><span style="font-weight: bold;">xor</span><br style="font-weight: bold;">
<br>
<span style="font-family: monospace;">Arguments: Two boolean values</span><br style="font-family: monospace;">
<span style="font-family: monospace;">Returns:&nbsp;&nbsp; Returns $(true) if exactly one of the booleans is true</span><br style="font-family: monospace;">
<hr style="width: 100%; height: 2px; font-family: monospace;"><span style="font-weight: bold;">nand</span><br>
<br>
<span style="font-family: monospace;">Arguments: Two boolean values</span><br style="font-family: monospace;">
<span style="font-family: monospace;">Returns:&nbsp;&nbsp; Returns value of 'not and'</span><br style="font-family: monospace;">
<hr style="width: 100%; height: 2px; font-family: monospace;"><span style="font-weight: bold;">nor</span><br>
<br>
<span style="font-family: monospace;">Arguments: Two boolean values</span><br style="font-family: monospace;">
<span style="font-family: monospace;">Returns:&nbsp;&nbsp; Returns value of 'not or'</span><br style="font-family: monospace;">
<hr style="width: 100%; height: 2px; font-family: monospace;"><span style="font-weight: bold;">xnor</span><br>
<br>
<span style="font-family: monospace;">Arguments: Two boolean values</span><br style="font-family: monospace;">
<span style="font-family: monospace;">Returns:&nbsp;&nbsp; Returns value of 'not xor'</span><br style="font-family: monospace;">
<hr style="width: 100%; height: 2px; font-family: monospace;">
<h2>List Manipulation Functions</h2>
&nbsp;A list is a string of characters; the list separator is a space.<br>
<br>
<hr style="width: 100%; height: 2px;"><b>first</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A list<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the first element of a list<br>
</span>
<hr><b>last</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A list<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the last element of a list<br>
</span>
<hr><b>rest</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A list<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the list with the first element
removed<br>
</span>
<hr><b>chop</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A list<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the list with the last element removed<br>
</span>
<hr><b>map</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of function to
$(call) for each element of list<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: List to
iterate over calling the function in 1<br>
Returns:&nbsp;&nbsp;&nbsp;The list after calling the function on each
element<br>
</span>
<hr><b>pairmap</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of function to
$(call) for each pair of elements<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: List to
iterate over calling the function in 1<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 3: Second
list to iterate over calling the function in 1<br>
Returns:&nbsp;&nbsp;&nbsp;The list after calling the function on each
pair of elements<br>
</span>
<hr><b>leq</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A list to compare
against...<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: ...this
list<br>
Returns:&nbsp;&nbsp;&nbsp;Returns $(true) if the two lists are identical<br>
</span>
<hr><b>lne</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A list to compare
against...<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: ...this
list<br>
Returns:&nbsp;&nbsp;&nbsp;Returns $(true) if the two lists are different<br>
</span>
<hr><b>reverse</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A list to reverse<br>
Returns:&nbsp;&nbsp;&nbsp;The list with its elements in reverse order<br>
</span>
<hr><b>uniq</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A list to deduplicate<br>
Returns:&nbsp;&nbsp;&nbsp;The list with elements in order without duplicates<br>
</span>
<hr><b>length</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A list<br>
Returns:&nbsp;&nbsp;&nbsp;The number of elements in the list<br>
</span>
<hr style="width: 100%; height: 2px;"><span style="font-family: monospace;"></span>
<h2>String Manipulation Functions</h2>
A string is any sequence of characters.<br>
<br>
<hr style="width: 100%; height: 2px;"><b>seq</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A string to compare
against...<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: ...this
string<br>
Returns:&nbsp;&nbsp;&nbsp;Returns $(true) if the two strings are
identical<br>
</span>
<hr><b>sne</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A string to compare
against...<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: ...this
string<br>
Returns:&nbsp;&nbsp;&nbsp;Returns $(true) if the two strings are not
the same<br>
</span>
<hr><b>strlen</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A string<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the length of the string<br>
</span>
<hr><b>substr</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A string<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Start offset (first character is 1)<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 3: Ending offset (inclusive)<br>Returns:&nbsp;&nbsp;&nbsp;Returns a substring<br>
</span>
<hr><b>split</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: The character to
split on<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: A
string to split<br>
Returns:&nbsp;&nbsp;&nbsp;Splits a string into a list separated by
spaces at the split<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; character
in the first argument<br>
</span>
<hr><b>merge</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: The character to
put between fields<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: A list
to merge into a string<br>
Returns:&nbsp;&nbsp;&nbsp;Merges a list into a single string, list
elements are separated<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; by the
character in the first argument<br>
</span>
<hr><b>tr</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: The list of
characters to translate from <br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: The
list of characters to translate to<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 3: The
text to translate<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the text after translating characters<br>
</span>
<hr><b>uc</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Text to upper case<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the text in upper case<br>
</span>
<hr><b>lc</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Text to lower case<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the text in lower case<br>
</span>
<hr style="width: 100%; height: 2px;"><span style="font-family: monospace;"></span>
<h2>Set Manipulation Functions</h2>
Sets are represented by sorted, deduplicated lists. To create a set
from a list use <span style="font-family:
monospace;">set_create</span>, or start with the <span
style="font-family: monospace;">empty_set</span> and <span
style="font-family: monospace;">set_insert</span> individual elements.
The empty set is defined as <span style="font-family:
monospace;">empty_set</span>.<p>
<hr><b>set_create</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A list of set elements<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the newly created set<br>
</span>
<hr><b>set_insert</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A single element to add to a set<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;2: A set<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the set with the element added<br>
</span>
<hr><b>set_remove</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A single element to remove from a set<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;2: A set<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the set with the element removed<br>
</span>
<hr><b>set_is_member</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A single element<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;2: A set<br>
Returns:&nbsp;&nbsp;&nbsp;Returns $(true) if the element is in the set<br>
</span>
<hr><b>set_is_not_member</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A single element<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;2: A set<br>
Returns:&nbsp;&nbsp;&nbsp;Returns $(false) if the element is in the set<br>
</span>
<hr><b>set_union</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A set<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;2: Another set<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the union of the two sets<br>
</span>
<hr><b>set_intersection</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A set<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;2: Another set<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the intersection of the two sets<br>
</span>
<hr><b>set_is_subset</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A set<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;2: Another set<br>
Returns:&nbsp;&nbsp;&nbsp;Returns $(true) if the first set is a subset of the second<br>
</span>
<hr><b>set_equal</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A set<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;2: Another set<br>
Returns:&nbsp;&nbsp;&nbsp;Returns $(true) if the two sets are identical<br>
</span>
<hr style="width: 100%; height: 2px;"><span style="font-family: monospace;"></span>
<h2>Integer Arithmetic Functions</h2>
Integers are represented by lists with the equivalent number of
x's.&nbsp; For example the number 4 is x x x x.&nbsp; The maximum
integer that the library can handle as <span style="font-style: italic;">input</span> (i.e. as the argument to a
call to <span style="font-family: monospace;">int_encode</span>) is
65536. There is no limit on integer size for internal computations or
output.<br>
<br>
The arithmetic library functions come in two forms: one form of each
function takes integers as arguments and the other form takes the
encoded form (x's created by a call to <span style="font-family: monospace;">int_encode</span>).&nbsp; For example,
there are two plus functions: <span style="font-family: monospace;">plus</span>
(called with integer arguments and returns an integer) and <span style="font-family: monospace;">int_plus</span> (called with encoded
arguments and returns an encoded result).<br>
<br>
<span style="font-family: monospace;">plus</span> will be slower than <span style="font-family: monospace;">int_plus</span> because its arguments
and result have to be translated between the x's format and
integers.&nbsp; If doing a complex calculation use the <span style="font-family: monospace;">int_*</span> forms with a single
encoding of inputs and single decoding of the output.&nbsp; For simple
calculations the direct forms can be used.<br>
<br>
<hr style="width: 100%; height: 2px;"><b>int_decode</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number of x's
representation<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the integer for human consumption
that is represented<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; by the
string of x's<br>
</span>
<hr><b>int_encode</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number in
human-readable integer form<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the integer encoded as a string of x's<br>
</span>
<hr><b>int_plus</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number in x's
representation<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
number in x's represntation<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the sum of the two numbers in x's
representation<br>
</span>
<hr><b>plus (wrapped version of int_plus)</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
integer<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the sum of the two integers<br>
</span>
<hr><b>int_subtract</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number in x's
representation<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
number in x's represntation<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the difference of the two numbers in
x's representation,<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; or outputs
an error on a numeric underflow<br>
</span>
<hr><b>subtract (wrapped version of int_subtract)</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
integer<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the difference of the two integers,<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; or outputs
an error on a numeric underflow<br>
</span>
<hr><b>int_multiply</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number in x's
representation<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
number in x's represntation<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the product of the two numbers in x's
representation<br>
</span>
<hr><b>multiply (wrapped version of int_multiply)</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
integer<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the product of the two integers<br>
</span>
<hr><b>int_divide</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number in x's
representation<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
number in x's represntation<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the result of integer division of
argument 1 divided<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; by
argument 2 in x's representation<br>
</span>
<hr><b>divide (wrapped version of int_divide)</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
integer<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the integer division of the first
argument by the second<br>
</span>
<hr><b>int_modulo</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number in x's
representation<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
number in x's represntation<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the remainder of integer division of
argument 1 divided<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; by
argument 2 in x's representation<br>
</span>
<hr><b>modulo (wrapped version of int_modulo)</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
integer<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the remainder of integer division of the first
argument by the second<br>
</span>
<hr><b>int_max, int_min</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number in x's
representation<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
number in x's represntation<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the maximum or minimum of its
arguments in x's<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
representation<br>
</span>
<hr><b>max, min</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Another
integer<br>
Returns:&nbsp;&nbsp;&nbsp;Returns the maximum or minimum of its integer
arguments<br>
</span>
<hr><b>int_gt, int_gte, int_lt, int_lte, int_eq, int_ne</b><br>
<br>
<span style="font-family: monospace;">Arguments: Two x's representation
numbers to be compared<br>
Returns:&nbsp;&nbsp;&nbsp;$(true) or $(false)<br>
<br>
int_gt First argument greater than second argument<br>
int_gte First argument greater than or equal to second argument<br>
int_lt First argument less than second argument <br>
int_lte First argument less than or equal to second argument<br>
int_eq First argument is numerically equal to the second argument<br>
int_ne First argument is not numerically equal to the second argument<br>
</span>
<hr><b>gt, gte, lt, lte, eq, ne</b><br>
<br>
<span style="font-family: monospace;">Arguments: Two integers to be
compared<br>
Returns:&nbsp;&nbsp;&nbsp;$(true) or $(false)<br>
<br>
gt First argument greater than second argument<br>
gte First argument greater than or equal to second argument<br>
lt First argument less than second argument <br>
lte First argument less than or equal to second argument<br>
eq First argument is numerically equal to the second argument<br>
ne First argument is not numerically equal to the second argument<br>
</span>
increment adds 1 to its argument, decrement subtracts 1. Note that<br>
decrement does not range check and hence will not underflow, but<br>
will incorrectly say that 0 - 1 = 0<br>
<hr><b>int_inc</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number in x's
representation<br>
Returns:&nbsp;&nbsp;&nbsp;The number incremented by 1 in x's
representation<br>
</span>
<hr><b>inc</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
Returns:&nbsp;&nbsp;&nbsp;The argument incremented by 1<br>
</span>
<hr><b>int_dec</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number in x's
representation<br>
Returns:&nbsp;&nbsp;&nbsp;The number decremented by 1 in x's
representation<br>
</span>
<hr><b>dec</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
Returns:&nbsp;&nbsp;&nbsp;The argument decremented by 1<br>
</span>
<hr><b>int_double</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number in x's
representation<br>
Returns:&nbsp;&nbsp;&nbsp;The number doubled (i.e. * 2) and returned in
x's representation<br>
</span>
<hr><b>double</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
Returns:&nbsp;&nbsp;&nbsp;The integer times 2<br>
</span>
<hr><b>int_halve</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A number in x's
representation<br>
Returns:&nbsp;&nbsp;&nbsp;The number halved (i.e. / 2) and returned in
x's representation<br>
</span>
<hr><b>halve</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
Returns:&nbsp;&nbsp;&nbsp;The integer divided by 2<br>
</span>
<hr><b>sequence</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: An integer<br>
Returns:&nbsp;&nbsp;&nbsp;The sequence [arg1 arg2] if arg1 >= arg2 or [arg2 arg1] if arg2 > arg1<br>
</span>
<hr><b>dec2hex, dec2bin, dec2oct</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: An integer<br>
Returns:&nbsp;&nbsp;&nbsp;The decimal argument converted to hexadecimal, binary or octal<br>
</span>
<hr style="width: 100%; height: 2px;"><span style="font-family: monospace;"></span>
<h2>Associative Arrays</h2>
An associate array maps a key value (a string with no spaces in it) to
a single value (any string).&nbsp;&nbsp;&nbsp; <br>
<b><br>
</b>
<hr style="width: 100%; height: 2px;"><b>set</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of associative
array<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: The key
value to associate<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 3: The
value associated with the key<br>
Returns:&nbsp;&nbsp;&nbsp;Nothing<br>
</span>
<hr><b>get</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of associative
array<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: The key
to retrieve<br>
Returns:&nbsp;&nbsp;&nbsp;The value stored in the array for that key<br>
</span>
<hr><b>keys</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of associative
array<br>
Returns:&nbsp;&nbsp;&nbsp;Returns a list of all defined keys in the
array<br>
</span>
<hr><b>defined</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of associative
array<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: The key
to test<br>
Returns:&nbsp;&nbsp;&nbsp;Returns true if the key is defined (i.e. not
empty)<br>
</span>
<hr style="width: 100%; height: 2px;"><span style="font-family: monospace;"></span>
<h2>Named Stacks</h2>
A stack is an ordered list of strings (with no spaces in them).<br>
<br>
<hr style="width: 100%; height: 2px;"><b>push</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of stack<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: Value
to push onto the top of the stack (must not contain<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; a space)<br>
Returns:&nbsp;&nbsp;&nbsp;None<br>
</span>
<hr><b>pop</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of stack<br>
Returns:&nbsp;&nbsp;&nbsp;Top element from the stack after removing it<br>
</span>
<hr><b>peek</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of stack<br>
Returns:&nbsp;&nbsp;&nbsp;Top element from the stack without removing it<br>
</span>
<hr><b>depth</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of stack<br>
Returns:&nbsp;&nbsp;&nbsp;Number of items on the stack<br>
</span>
<hr style="width: 100%; height: 2px;"><span style="font-family: monospace;"></span>
<h2>Function memoization</h2>
To reduce the number of calls to slow functions (such as $(shell) a single memoization function is provided.<br>
<br>
<hr style="width: 100%; height: 2px;"><b>memoize</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of function to memoize<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: String argument for the function<br>
Returns:&nbsp;&nbsp;&nbsp;Result of $1 applied to $2 but only calls $1 once for each unique $2<br>
</span>
<hr style="width: 100%; height: 2px;"><span style="font-family: monospace;"></span>
<h2>Miscellaneous and Debugging Facilities</h2>
GMSL defines the following constants; all are accessed as normal GNU
Make variables by wrapping them in <span style="font-family: monospace;">$()</span> or <span style="font-family: monospace;">${}</span>.<br>
<br>
<table style="text-align: left;" border="1" cellpadding="2" cellspacing="2">
<tbody>
<tr>
<td><span style="font-style: italic;">Constant</span><br>
</td>
<td><span style="font-style: italic;">Value</span><br>
</td>
<td><span style="font-style: italic;">Purpose</span><br>
</td>
</tr>
<tr>
<td><span style="font-family: monospace;">true</span><br>
</td>
<td><span style="font-family: monospace;">T</span><br>
</td>
<td>Boolean for <span style="font-family: monospace;">$(if)</span>
and return from&nbsp; GMSL functions<br>
</td>
</tr>
<tr>
<td><span style="font-family: monospace;">false</span><br>
</td>
<td><br>
</td>
<td>Boolean for <span style="font-family: monospace;">$(if)</span>
and return from GMSL functions<br>
</td>
</tr>
<tr>
<td><span style="font-family: monospace;">gmsl_version</span><br>
</td>
<td><span style="font-family: monospace;">1 0 0</span><br>
</td>
<td>GMSL version number as list: major minor revision<br>
</td>
</tr>
</tbody>
</table>
<span style="font-weight: bold;"><br>
gmsl_compatible</span><span style="font-family: monospace;"><br>
<br>
Arguments: List containing the desired library version number (maj min
rev)<br>
</span><span style="font-family: monospace;">Returns:&nbsp;&nbsp;
$(true) if this version of the library is compatible<br>
</span><span style="font-family: monospace;">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
with the requested version number, otherwise $(false)</span>
<hr><b>gmsl-print-% (target not a function)</b><br>
<br>
<span style="font-family: monospace;">Arguments: The % should be
replaced by the name of a variable that you<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; wish to
print out.<br>
Action:&nbsp;&nbsp;&nbsp; Echos the name of the variable that matches
the % and its value.<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; For
example, 'make gmsl-print-SHELL' will output the value of<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; the SHELL
variable<br>
</span>
<hr><b>gmsl-echo-% (target not a function)</b><br>
<br>
<span style="font-family: monospace;">Arguments: The % should be
replaced by the name of a variable that you<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; wish to
print out.<br>
Action:&nbsp;&nbsp;&nbsp; Echos the value of the variable that matches
the %.<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; For
example, 'make gmsl-echo-SHELL' will output the value of<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; the SHELL
variable<br>
</span>
<hr><b>assert</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: A boolean that must
be true or the assertion will fail<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2: The
message to print with the assertion<br>
Returns:&nbsp;&nbsp;&nbsp;None<br>
</span>
<hr><b>assert_exists</b><br>
<br>
<span style="font-family: monospace;">Arguments: 1: Name of file that
must exist, if it is missing an assertion<br>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; will be
generated<br>
Returns:&nbsp;&nbsp;&nbsp;None<br>
</span>
<hr style="width: 100%; height: 2px;"><br>
GMSL has a number of environment variables (or command-line overrides)
that control various bits of functionality:<br>
<br>
<table style="text-align: left;" border="1" cellpadding="2" cellspacing="2">
<tbody>
<tr>
<td><span style="font-style: italic;">Variable</span><br>
</td>
<td><span style="font-style: italic;">Purpose</span><br>
</td>
</tr>
<tr>
<td><span style="font-family: monospace;">GMSL_NO_WARNINGS</span><br>
</td>
<td>If set prevents GMSL from outputting warning messages:
artithmetic functions generate underflow warnings.<br>
</td>
</tr>
<tr>
<td><span style="font-family: monospace;">GMSL_NO_ERRORS</span><br>
</td>
<td>If set prevents GMSL from generating fatal errors: division
by zero or failed assertions are fatal.<br>
</td>
</tr>
<tr>
<td><span style="font-family: monospace;">GMSL_TRACE</span><br>
</td>
<td>Enables function tracing.&nbsp; Calls to GMSL functions will
result in name and arguments being traced.<br>
</td>
</tr>
</tbody>
</table>
<span style="font-family: monospace;"></span><br>
<hr>
Copyright (c) 2005-2022 <a href="https://www.jgc.org/">John Graham-Cumming</a>.<br>
<hr style="width: 100%; height: 2px;">
</body></html>

8
mk/platform/unix_common.mk
View file

@ -7,14 +7,8 @@ MKDIR=mkdir -p
# Go should not be required to run functional tests
GOOS ?= $(shell go env GOOS)
# Current versioning information from env
#Current versioning information from env
BUILD_VERSION?=$(shell git describe --tags)
BUILD_TIMESTAMP=$(shell date +%F"_"%T)
DEFAULT_CONFIGDIR?=/etc/crowdsec
DEFAULT_DATADIR?=/var/lib/crowdsec/data
PKG_CONFIG:=$(shell command -v pkg-config 2>/dev/null)
# See if we have libre2-dev installed for C++ optimizations.
# In fedora and other distros, we need to tell where to find re2.pc
RE2_CHECK := $(shell PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$(PKG_CONFIG_PATH) pkg-config --libs re2 2>/dev/null)

8
mk/platform/windows.mk
View file

@ -4,9 +4,9 @@ MAKE=make
GOOS=windows
PREFIX=$(shell $$env:TEMP)
# Current versioning information from env
# BUILD_VERSION?=$(shell (Invoke-WebRequest -UseBasicParsing -Uri https://api.github.com/repos/crowdsecurity/crowdsec/releases/latest).Content | jq -r '.tag_name')
# hardcode it till I find a workaround
#Current versioning information from env
#BUILD_VERSION?=$(shell (Invoke-WebRequest -UseBasicParsing -Uri https://api.github.com/repos/crowdsecurity/crowdsec/releases/latest).Content | jq -r '.tag_name')
#hardcode it till i find a workaround
BUILD_VERSION?=$(shell git describe --tags $$(git rev-list --tags --max-count=1))
BUILD_TIMESTAMP?=$(shell Get-Date -Format "yyyy-MM-dd_HH:mm:ss")
DEFAULT_CONFIGDIR?=C:\\ProgramData\\CrowdSec\\config
@ -18,5 +18,3 @@ CP=Copy-Item
CPR=Copy-Item -Recurse
MKDIR=New-Item -ItemType directory
WIN_IGNORE_ERR=; exit 0
PKG_CONFIG:=$(shell Get-Command pkg-config -ErrorAction SilentlyContinue)

46
pkg/acquisition/acquisition.go
View file

@ -35,20 +35,6 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/types"
)
type DataSourceUnavailableError struct {
Name string
Err error
}
func (e *DataSourceUnavailableError) Error() string {
return fmt.Sprintf("datasource '%s' is not available: %v", e.Name, e.Err)
}
func (e *DataSourceUnavailableError) Unwrap() error {
return e.Err
}
// The interface each datasource must implement
type DataSource interface {
GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module
@ -89,10 +75,6 @@ func GetDataSourceIface(dataSourceType string) DataSource {
return source()
}
// DataSourceConfigure creates and returns a DataSource object from a configuration,
// if the configuration is not valid it returns an error.
// If the datasource can't be run (eg. journalctl not available), it still returns an error which
// can be checked for the appropriate action.
func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataSource, error) {
// we dump it back to []byte, because we want to decode the yaml blob twice:
// once to DataSourceCommonCfg, and then later to the dedicated type of the datasource
@ -118,7 +100,7 @@ func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataS
subLogger := clog.WithFields(customLog)
/* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */
if err := dataSrc.CanRun(); err != nil {
return nil, &DataSourceUnavailableError{Name: commonConfig.Source, Err: err}
return nil, fmt.Errorf("datasource %s cannot be run: %w", commonConfig.Source, err)
}
/* configure the actual datasource */
if err := dataSrc.Configure(yamlConfig, subLogger); err != nil {
@ -191,11 +173,10 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
}
dec := yaml.NewDecoder(yamlFile)
dec.SetStrict(true)
idx := -1
for {
var sub configuration.DataSourceCommonCfg
var idx int
err = dec.Decode(&sub)
idx += 1
if err != nil {
if !errors.Is(err, io.EOF) {
return nil, fmt.Errorf("failed to yaml decode %s: %w", acquisFile, err)
@ -212,6 +193,7 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
if len(sub.Labels) == 0 {
if sub.Source == "" {
log.Debugf("skipping empty item in %s", acquisFile)
idx += 1
continue
}
return nil, fmt.Errorf("missing labels in %s (position: %d)", acquisFile, idx)
@ -226,11 +208,6 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
sub.UniqueId = uniqueId
src, err := DataSourceConfigure(sub)
if err != nil {
var dserr *DataSourceUnavailableError
if errors.As(err, &dserr) {
log.Error(err)
continue
}
return nil, fmt.Errorf("while configuring datasource of type %s from %s (position: %d): %w", sub.Source, acquisFile, idx, err)
}
if sub.TransformExpr != "" {
@ -241,6 +218,7 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
transformRuntimes[uniqueId] = vm
}
sources = append(sources, *src)
idx += 1
}
}
return sources, nil
@ -317,11 +295,6 @@ func transform(transformChan chan types.Event, output chan types.Event, AcquisTo
}
func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb *tomb.Tomb) error {
// Don't wait if we have no sources, as it will hang forever
if len(sources) == 0 {
return nil
}
for i := 0; i < len(sources); i++ {
subsrc := sources[i] //ensure its a copy
log.Debugf("starting one source %d/%d ->> %T", i, len(sources), subsrc)
@ -357,8 +330,11 @@ func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb
return nil
})
}
/*return only when acquisition is over (cat) or never (tail)*/
err := AcquisTomb.Wait()
return err
// Don't wait if we have no sources, as it will hang forever
if len(sources) > 0 {
/*return only when acquisition is over (cat) or never (tail)*/
err := AcquisTomb.Wait()
return err
}
return nil
}

2
pkg/acquisition/acquisition_test.go
View file

@ -167,7 +167,7 @@ log_level: debug
source: mock_cant_run
wowo: ajsajasjas
`,
ExpectedError: "datasource 'mock_cant_run' is not available: can't run bro",
ExpectedError: "datasource mock_cant_run cannot be run: can't run bro",
},
}

16
pkg/acquisition/modules/cloudwatch/cloudwatch.go
View file

@ -13,6 +13,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
@ -200,7 +201,7 @@ func (cw *CloudwatchSource) Configure(yamlConfig []byte, logger *log.Entry) erro
targetStream := "*"
if cw.Config.StreamRegexp != nil {
if _, err := regexp.Compile(*cw.Config.StreamRegexp); err != nil {
return fmt.Errorf("while compiling regexp '%s': %w", *cw.Config.StreamRegexp, err)
return errors.Wrapf(err, "error while compiling regexp '%s'", *cw.Config.StreamRegexp)
}
targetStream = *cw.Config.StreamRegexp
} else if cw.Config.StreamName != nil {
@ -344,7 +345,8 @@ func (cw *CloudwatchSource) WatchLogGroupForStreams(out chan LogStreamTailConfig
},
)
if err != nil {
return fmt.Errorf("while describing group %s: %w", cw.Config.GroupName, err)
newerr := errors.Wrapf(err, "while describing group %s", cw.Config.GroupName)
return newerr
}
cw.logger.Tracef("after DescribeLogStreamsPagesWithContext")
}
@ -493,7 +495,7 @@ func (cw *CloudwatchSource) TailLogStream(cfg *LogStreamTailConfig, outChan chan
},
)
if err != nil {
newerr := fmt.Errorf("while reading %s/%s: %w", cfg.GroupName, cfg.StreamName, err)
newerr := errors.Wrapf(err, "while reading %s/%s", cfg.GroupName, cfg.StreamName)
cfg.logger.Warningf("err : %s", newerr)
return newerr
}
@ -530,7 +532,7 @@ func (cw *CloudwatchSource) ConfigureByDSN(dsn string, labels map[string]string,
u, err := url.ParseQuery(args[1])
if err != nil {
return fmt.Errorf("while parsing %s: %w", dsn, err)
return errors.Wrapf(err, "while parsing %s", dsn)
}
for k, v := range u {
@ -541,7 +543,7 @@ func (cw *CloudwatchSource) ConfigureByDSN(dsn string, labels map[string]string,
}
lvl, err := log.ParseLevel(v[0])
if err != nil {
return fmt.Errorf("unknown level %s: %w", v[0], err)
return errors.Wrapf(err, "unknown level %s", v[0])
}
cw.logger.Logger.SetLevel(lvl)
@ -575,7 +577,7 @@ func (cw *CloudwatchSource) ConfigureByDSN(dsn string, labels map[string]string,
//let's reuse our parser helper so that a ton of date formats are supported
duration, err := time.ParseDuration(v[0])
if err != nil {
return fmt.Errorf("unable to parse '%s' as duration: %w", v[0], err)
return errors.Wrapf(err, "unable to parse '%s' as duration", v[0])
}
cw.logger.Debugf("parsed '%s' as '%s'", v[0], duration)
start := time.Now().UTC().Add(-duration)
@ -672,7 +674,7 @@ func (cw *CloudwatchSource) CatLogStream(cfg *LogStreamTailConfig, outChan chan
},
)
if err != nil {
return fmt.Errorf("while reading logs from %s/%s: %w", cfg.GroupName, cfg.StreamName, err)
return errors.Wrapf(err, "while reading logs from %s/%s", cfg.GroupName, cfg.StreamName)
}
cfg.logger.Tracef("after GetLogEventsPagesWithContext")
case <-cw.t.Dying():

7
pkg/acquisition/modules/docker/docker.go
View file

@ -12,6 +12,7 @@ import (
dockerTypes "github.com/docker/docker/api/types"
"github.com/docker/docker/client"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
@ -79,7 +80,7 @@ func (d *DockerSource) UnmarshalConfig(yamlConfig []byte) error {
err := yaml.UnmarshalStrict(yamlConfig, &d.Config)
if err != nil {
return fmt.Errorf("while parsing DockerAcquisition configuration: %w", err)
return errors.Wrap(err, "Cannot parse DockerAcquisition configuration")
}
if d.logger != nil {
@ -213,7 +214,7 @@ func (d *DockerSource) ConfigureByDSN(dsn string, labels map[string]string, logg
parameters, err := url.ParseQuery(args[1])
if err != nil {
return fmt.Errorf("while parsing parameters %s: %w", dsn, err)
return errors.Wrapf(err, "while parsing parameters %s: %s", dsn, err)
}
for k, v := range parameters {
@ -224,7 +225,7 @@ func (d *DockerSource) ConfigureByDSN(dsn string, labels map[string]string, logg
}
lvl, err := log.ParseLevel(v[0])
if err != nil {
return fmt.Errorf("unknown level %s: %w", v[0], err)
return errors.Wrapf(err, "unknown level %s", v[0])
}
d.logger.Logger.SetLevel(lvl)
case "until":

22
pkg/acquisition/modules/file/file.go
View file

@ -14,6 +14,8 @@ import (
"strings"
"time"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/fsnotify/fsnotify"
"github.com/nxadm/tail"
"github.com/pkg/errors"
@ -22,8 +24,6 @@ import (
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
@ -110,7 +110,7 @@ func (f *FileSource) Configure(yamlConfig []byte, logger *log.Entry) error {
f.watcher, err = fsnotify.NewWatcher()
if err != nil {
return fmt.Errorf("could not create fsnotify watcher: %w", err)
return errors.Wrapf(err, "Could not create fsnotify watcher")
}
f.logger.Tracef("Actual FileAcquisition Configuration %+v", f.config)
@ -130,7 +130,7 @@ func (f *FileSource) Configure(yamlConfig []byte, logger *log.Entry) error {
}
files, err := filepath.Glob(pattern)
if err != nil {
return fmt.Errorf("glob failure: %w", err)
return errors.Wrap(err, "Glob failure")
}
if len(files) == 0 {
f.logger.Warnf("No matching files for pattern %s", pattern)
@ -191,7 +191,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger
if len(args) == 2 && len(args[1]) != 0 {
params, err := url.ParseQuery(args[1])
if err != nil {
return fmt.Errorf("could not parse file args: %w", err)
return errors.Wrap(err, "could not parse file args")
}
for key, value := range params {
switch key {
@ -201,7 +201,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger
}
lvl, err := log.ParseLevel(value[0])
if err != nil {
return fmt.Errorf("unknown level %s: %w", value[0], err)
return errors.Wrapf(err, "unknown level %s", value[0])
}
f.logger.Logger.SetLevel(lvl)
case "max_buffer_size":
@ -210,7 +210,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger
}
maxBufferSize, err := strconv.Atoi(value[0])
if err != nil {
return fmt.Errorf("could not parse max_buffer_size %s: %w", value[0], err)
return errors.Wrapf(err, "could not parse max_buffer_size %s", value[0])
}
f.config.MaxBufferSize = maxBufferSize
default:
@ -226,7 +226,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger
f.logger.Debugf("Will try pattern %s", args[0])
files, err := filepath.Glob(args[0])
if err != nil {
return fmt.Errorf("glob failure: %w", err)
return errors.Wrap(err, "Glob failure")
}
if len(files) == 0 {
@ -433,7 +433,7 @@ func (f *FileSource) monitorNewFiles(out chan types.Event, t *tomb.Tomb) error {
case <-t.Dying():
err := f.watcher.Close()
if err != nil {
return fmt.Errorf("could not remove all inotify watches: %w", err)
return errors.Wrapf(err, "could not remove all inotify watches")
}
return nil
}
@ -495,7 +495,7 @@ func (f *FileSource) readFile(filename string, out chan types.Event, t *tomb.Tom
fd, err := os.Open(filename)
if err != nil {
return fmt.Errorf("failed opening %s: %w", filename, err)
return errors.Wrapf(err, "failed opening %s", filename)
}
defer fd.Close()
@ -503,7 +503,7 @@ func (f *FileSource) readFile(filename string, out chan types.Event, t *tomb.Tom
gz, err := gzip.NewReader(fd)
if err != nil {
logger.Errorf("Failed to read gz file: %s", err)
return fmt.Errorf("failed to read gz %s: %w", filename, err)
return errors.Wrapf(err, "failed to read gz %s", filename)
}
defer gz.Close()
scanner = bufio.NewScanner(gz)

4
pkg/acquisition/modules/file/file_test.go
View file

@ -38,7 +38,7 @@ func TestBadConfiguration(t *testing.T) {
{
name: "glob syntax error",
config: `filename: "[asd-.log"`,
expectedErr: "glob failure: syntax error in pattern",
expectedErr: "Glob failure: syntax error in pattern",
},
{
name: "bad exclude regexp",
@ -150,7 +150,7 @@ filename: /`,
config: `
mode: cat
filename: "[*-.log"`,
expectedConfigErr: "glob failure: syntax error in pattern",
expectedConfigErr: "Glob failure: syntax error in pattern",
logLevel: log.WarnLevel,
expectedLines: 0,
},

3
pkg/acquisition/modules/journalctl/journalctl.go
View file

@ -9,6 +9,7 @@ import (
"strings"
"time"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
@ -236,7 +237,7 @@ func (j *JournalCtlSource) ConfigureByDSN(dsn string, labels map[string]string,
}
lvl, err := log.ParseLevel(value[0])
if err != nil {
return fmt.Errorf("unknown level %s: %w", value[0], err)
return errors.Wrapf(err, "unknown level %s", value[0])
}
j.logger.Logger.SetLevel(lvl)
case "since":

11
pkg/acquisition/modules/kafka/kafka.go
View file

@ -10,6 +10,7 @@ import (
"strconv"
"time"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/segmentio/kafka-go"
log "github.com/sirupsen/logrus"
@ -92,12 +93,12 @@ func (k *KafkaSource) Configure(yamlConfig []byte, logger *log.Entry) error {
dialer, err := k.Config.NewDialer()
if err != nil {
return fmt.Errorf("cannot create %s dialer: %w", dataSourceName, err)
return errors.Wrapf(err, "cannot create %s dialer", dataSourceName)
}
k.Reader, err = k.Config.NewReader(dialer)
if err != nil {
return fmt.Errorf("cannote create %s reader: %w", dataSourceName, err)
return errors.Wrapf(err, "cannote create %s reader", dataSourceName)
}
if k.Reader == nil {
@ -148,7 +149,7 @@ func (k *KafkaSource) ReadMessage(out chan types.Event) error {
if err == io.EOF {
return nil
}
k.logger.Errorln(fmt.Errorf("while reading %s message: %w", dataSourceName, err))
k.logger.Errorln(errors.Wrapf(err, "while reading %s message", dataSourceName))
}
l := types.Line{
Raw: string(m.Value),
@ -180,7 +181,7 @@ func (k *KafkaSource) RunReader(out chan types.Event, t *tomb.Tomb) error {
case <-t.Dying():
k.logger.Infof("%s datasource topic %s stopping", dataSourceName, k.Config.Topic)
if err := k.Reader.Close(); err != nil {
return fmt.Errorf("while closing %s reader on topic '%s': %w", dataSourceName, k.Config.Topic, err)
return errors.Wrapf(err, "while closing %s reader on topic '%s'", dataSourceName, k.Config.Topic)
}
return nil
}
@ -263,7 +264,7 @@ func (kc *KafkaConfiguration) NewReader(dialer *kafka.Dialer) (*kafka.Reader, er
rConf.GroupID = kc.GroupID
}
if err := rConf.Validate(); err != nil {
return &kafka.Reader{}, fmt.Errorf("while validating reader configuration: %w", err)
return &kafka.Reader{}, errors.Wrapf(err, "while validating reader configuration")
}
return kafka.NewReader(rConf), nil
}

13
pkg/acquisition/modules/kafka/kafka_test.go
View file

@ -8,14 +8,13 @@ import (
"testing"
"time"
"github.com/segmentio/kafka-go"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
"gopkg.in/tomb.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/cstest"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/segmentio/kafka-go"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
"gotest.tools/v3/assert"
)
func TestConfigure(t *testing.T) {
@ -179,7 +178,7 @@ topic: crowdsecplaintext`), subLogger)
break READLOOP
}
}
require.Equal(t, ts.expectedLines, actualLines)
assert.Equal(t, ts.expectedLines, actualLines)
tomb.Kill(nil)
tomb.Wait()
})
@ -255,7 +254,7 @@ tls:
break READLOOP
}
}
require.Equal(t, ts.expectedLines, actualLines)
assert.Equal(t, ts.expectedLines, actualLines)
tomb.Kill(nil)
tomb.Wait()
})

37
pkg/acquisition/modules/kinesis/kinesis.go
View file

@ -13,6 +13,7 @@ import (
"github.com/aws/aws-sdk-go/aws/arn"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/kinesis"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
@ -123,7 +124,7 @@ func (k *KinesisSource) UnmarshalConfig(yamlConfig []byte) error {
err := yaml.UnmarshalStrict(yamlConfig, &k.Config)
if err != nil {
return fmt.Errorf("Cannot parse kinesis datasource configuration: %w", err)
return errors.Wrap(err, "Cannot parse kinesis datasource configuration")
}
if k.Config.Mode == "" {
@ -217,7 +218,7 @@ func (k *KinesisSource) WaitForConsumerDeregistration(consumerName string, strea
return nil
default:
k.logger.Errorf("Error while waiting for consumer deregistration: %s", err)
return fmt.Errorf("cannot describe stream consumer: %w", err)
return errors.Wrap(err, "Cannot describe stream consumer")
}
}
time.Sleep(time.Millisecond * 200 * time.Duration(i+1))
@ -235,12 +236,12 @@ func (k *KinesisSource) DeregisterConsumer() error {
switch err.(type) {
case *kinesis.ResourceNotFoundException:
default:
return fmt.Errorf("cannot deregister stream consumer: %w", err)
return errors.Wrap(err, "Cannot deregister stream consumer")
}
}
err = k.WaitForConsumerDeregistration(k.Config.ConsumerName, k.Config.StreamARN)
if err != nil {
return fmt.Errorf("cannot wait for consumer deregistration: %w", err)
return errors.Wrap(err, "Cannot wait for consumer deregistration")
}
return nil
}
@ -252,7 +253,7 @@ func (k *KinesisSource) WaitForConsumerRegistration(consumerARN string) error {
ConsumerARN: aws.String(consumerARN),
})
if err != nil {
return fmt.Errorf("cannot describe stream consumer: %w", err)
return errors.Wrap(err, "Cannot describe stream consumer")
}
if *describeOutput.ConsumerDescription.ConsumerStatus == "ACTIVE" {
k.logger.Debugf("Consumer %s is active", consumerARN)
@ -271,11 +272,11 @@ func (k *KinesisSource) RegisterConsumer() (*kinesis.RegisterStreamConsumerOutpu
StreamARN: aws.String(k.Config.StreamARN),
})
if err != nil {
return nil, fmt.Errorf("cannot register stream consumer: %w", err)
return nil, errors.Wrap(err, "Cannot register stream consumer")
}
err = k.WaitForConsumerRegistration(*streamConsumer.Consumer.ConsumerARN)
if err != nil {
return nil, fmt.Errorf("timeout while waiting for consumer to be active: %w", err)
return nil, errors.Wrap(err, "Timeout while waiting for consumer to be active")
}
return streamConsumer, nil
}
@ -338,7 +339,7 @@ func (k *KinesisSource) ReadFromSubscription(reader kinesis.SubscribeToShardEven
logger.Infof("Subscribed shard reader is dying")
err := reader.Close()
if err != nil {
return fmt.Errorf("cannot close kinesis subscribed shard reader: %w", err)
return errors.Wrap(err, "Cannot close kinesis subscribed shard reader")
}
return nil
case event, ok := <-reader.Events():
@ -361,7 +362,7 @@ func (k *KinesisSource) SubscribeToShards(arn arn.ARN, streamConsumer *kinesis.R
StreamName: aws.String(arn.Resource[7:]),
})
if err != nil {
return fmt.Errorf("cannot list shards for enhanced_read: %w", err)
return errors.Wrap(err, "Cannot list shards for enhanced_read")
}
for _, shard := range shards.Shards {
@ -372,7 +373,7 @@ func (k *KinesisSource) SubscribeToShards(arn arn.ARN, streamConsumer *kinesis.R
ConsumerARN: streamConsumer.Consumer.ConsumerARN,
})
if err != nil {
return fmt.Errorf("cannot subscribe to shard: %w", err)
return errors.Wrap(err, "Cannot subscribe to shard")
}
k.shardReaderTomb.Go(func() error {
return k.ReadFromSubscription(r.GetEventStream().Reader, out, shardId, arn.Resource[7:])
@ -384,7 +385,7 @@ func (k *KinesisSource) SubscribeToShards(arn arn.ARN, streamConsumer *kinesis.R
func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error {
parsedARN, err := arn.Parse(k.Config.StreamARN)
if err != nil {
return fmt.Errorf("cannot parse stream ARN: %w", err)
return errors.Wrap(err, "Cannot parse stream ARN")
}
if !strings.HasPrefix(parsedARN.Resource, "stream/") {
return fmt.Errorf("resource part of stream ARN %s does not start with stream/", k.Config.StreamARN)
@ -394,12 +395,12 @@ func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error {
k.logger.Info("starting kinesis acquisition with enhanced fan-out")
err = k.DeregisterConsumer()
if err != nil {
return fmt.Errorf("cannot deregister consumer: %w", err)
return errors.Wrap(err, "Cannot deregister consumer")
}
streamConsumer, err := k.RegisterConsumer()
if err != nil {
return fmt.Errorf("cannot register consumer: %w", err)
return errors.Wrap(err, "Cannot register consumer")
}
for {
@ -407,7 +408,7 @@ func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error {
err = k.SubscribeToShards(parsedARN, streamConsumer, out)
if err != nil {
return fmt.Errorf("cannot subscribe to shards: %w", err)
return errors.Wrap(err, "Cannot subscribe to shards")
}
select {
case <-t.Dying():
@ -416,7 +417,7 @@ func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error {
_ = k.shardReaderTomb.Wait() //we don't care about the error as we kill the tomb ourselves
err = k.DeregisterConsumer()
if err != nil {
return fmt.Errorf("cannot deregister consumer: %w", err)
return errors.Wrap(err, "Cannot deregister consumer")
}
return nil
case <-k.shardReaderTomb.Dying():
@ -439,7 +440,7 @@ func (k *KinesisSource) ReadFromShard(out chan types.Event, shardId string) erro
ShardIteratorType: aws.String(kinesis.ShardIteratorTypeLatest)})
if err != nil {
logger.Errorf("Cannot get shard iterator: %s", err)
return fmt.Errorf("cannot get shard iterator: %w", err)
return errors.Wrap(err, "Cannot get shard iterator")
}
it := sharIt.ShardIterator
//AWS recommends to wait for a second between calls to GetRecords for a given shard
@ -460,7 +461,7 @@ func (k *KinesisSource) ReadFromShard(out chan types.Event, shardId string) erro
continue
default:
logger.Error("Cannot get records")
return fmt.Errorf("cannot get records: %w", err)
return errors.Wrap(err, "Cannot get records")
}
}
k.ParseAndPushRecords(records.Records, out, logger, shardId)
@ -485,7 +486,7 @@ func (k *KinesisSource) ReadFromStream(out chan types.Event, t *tomb.Tomb) error
StreamName: aws.String(k.Config.StreamName),
})
if err != nil {
return fmt.Errorf("cannot list shards: %w", err)
return errors.Wrap(err, "Cannot list shards")
}
k.shardReaderTomb = &tomb.Tomb{}
for _, shard := range shards.Shards {

14
pkg/acquisition/modules/kubernetesaudit/k8s_audit.go
View file

@ -8,16 +8,16 @@ import (
"net/http"
"strings"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
"k8s.io/apiserver/pkg/apis/audit"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
type KubernetesAuditConfiguration struct {
@ -66,7 +66,7 @@ func (ka *KubernetesAuditSource) UnmarshalConfig(yamlConfig []byte) error {
k8sConfig := KubernetesAuditConfiguration{}
err := yaml.UnmarshalStrict(yamlConfig, &k8sConfig)
if err != nil {
return fmt.Errorf("cannot parse k8s-audit configuration: %w", err)
return errors.Wrap(err, "Cannot parse k8s-audit configuration")
}
ka.config = k8sConfig
@ -140,7 +140,7 @@ func (ka *KubernetesAuditSource) StreamingAcquisition(out chan types.Event, t *t
t.Go(func() error {
err := ka.server.ListenAndServe()
if err != nil && err != http.ErrServerClosed {
return fmt.Errorf("k8s-audit server failed: %w", err)
return errors.Wrap(err, "k8s-audit server failed")
}
return nil
})

13
pkg/acquisition/modules/s3/s3.go
View file

@ -6,7 +6,6 @@ import (
"compress/gzip"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/url"
@ -22,13 +21,13 @@ import (
"github.com/aws/aws-sdk-go/service/s3/s3iface"
"github.com/aws/aws-sdk-go/service/sqs"
"github.com/aws/aws-sdk-go/service/sqs/sqsiface"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
type S3Configuration struct {
@ -564,7 +563,7 @@ func (s *S3Source) ConfigureByDSN(dsn string, labels map[string]string, logger *
if len(args) == 2 && len(args[1]) != 0 {
params, err := url.ParseQuery(args[1])
if err != nil {
return fmt.Errorf("could not parse s3 args: %w", err)
return errors.Wrap(err, "could not parse s3 args")
}
for key, value := range params {
switch key {
@ -574,7 +573,7 @@ func (s *S3Source) ConfigureByDSN(dsn string, labels map[string]string, logger *
}
lvl, err := log.ParseLevel(value[0])
if err != nil {
return fmt.Errorf("unknown level %s: %w", value[0], err)
return errors.Wrapf(err, "unknown level %s", value[0])
}
s.logger.Logger.SetLevel(lvl)
case "max_buffer_size":
@ -583,7 +582,7 @@ func (s *S3Source) ConfigureByDSN(dsn string, labels map[string]string, logger *
}
maxBufferSize, err := strconv.Atoi(value[0])
if err != nil {
return fmt.Errorf("invalid value for 'max_buffer_size': %w", err)
return errors.Wrapf(err, "invalid value for 'max_buffer_size'")
}
s.logger.Debugf("Setting max buffer size to %d", maxBufferSize)
s.Config.MaxBufferSize = maxBufferSize

9
pkg/acquisition/modules/syslog/internal/server/syslogserver.go
View file

@ -6,6 +6,7 @@ import (
"strings"
"time"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
)
@ -30,18 +31,18 @@ func (s *SyslogServer) Listen(listenAddr string, port int) error {
s.port = port
udpAddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", s.listenAddr, s.port))
if err != nil {
return fmt.Errorf("could not resolve addr %s: %w", s.listenAddr, err)
return errors.Wrapf(err, "could not resolve addr %s", s.listenAddr)
}
udpConn, err := net.ListenUDP("udp", udpAddr)
if err != nil {
return fmt.Errorf("could not listen on port %d: %w", s.port, err)
return errors.Wrapf(err, "could not listen on port %d", s.port)
}
s.Logger.Debugf("listening on %s:%d", s.listenAddr, s.port)
s.udpConn = udpConn
err = s.udpConn.SetReadDeadline(time.Now().UTC().Add(100 * time.Millisecond))
if err != nil {
return fmt.Errorf("could not set read deadline on UDP socket: %w", err)
return errors.Wrap(err, "could not set read deadline on UDP socket")
}
return nil
}
@ -86,7 +87,7 @@ func (s *SyslogServer) StartServer() *tomb.Tomb {
func (s *SyslogServer) KillServer() error {
err := s.udpConn.Close()
if err != nil {
return fmt.Errorf("could not close UDP connection: %w", err)
return errors.Wrap(err, "could not close UDP connection")
}
close(s.channel)
return nil

5
pkg/acquisition/modules/syslog/syslog.go
View file

@ -6,6 +6,7 @@ import (
"strings"
"time"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
@ -99,7 +100,7 @@ func (s *SyslogSource) UnmarshalConfig(yamlConfig []byte) error {
err := yaml.UnmarshalStrict(yamlConfig, &s.config)
if err != nil {
return fmt.Errorf("cannot parse syslog configuration: %w", err)
return errors.Wrap(err, "Cannot parse syslog configuration")
}
if s.config.Addr == "" {
@ -139,7 +140,7 @@ func (s *SyslogSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb)
s.server.SetChannel(c)
err := s.server.Listen(s.config.Addr, s.config.Port)
if err != nil {
return fmt.Errorf("could not start syslog server: %w", err)
return errors.Wrap(err, "could not start syslog server")
}
s.serverTomb = s.server.StartServer()
t.Go(func() error {

10
pkg/apiclient/alerts_service.go
View file

@ -5,9 +5,9 @@ import (
"fmt"
"net/http"
qs "github.com/google/go-querystring/query"
"github.com/crowdsecurity/crowdsec/pkg/models"
qs "github.com/google/go-querystring/query"
"github.com/pkg/errors"
)
// type ApiAlerts service
@ -72,7 +72,7 @@ func (s *AlertsService) List(ctx context.Context, opts AlertsListOpts) (*models.
u := fmt.Sprintf("%s/alerts", s.client.URLPrefix)
params, err := qs.Values(opts)
if err != nil {
return nil, nil, fmt.Errorf("building query: %w", err)
return nil, nil, errors.Wrap(err, "building query")
}
if len(params) > 0 {
URI = fmt.Sprintf("%s?%s", u, params.Encode())
@ -82,12 +82,12 @@ func (s *AlertsService) List(ctx context.Context, opts AlertsListOpts) (*models.
req, err := s.client.NewRequest(http.MethodGet, URI, nil)
if err != nil {
return nil, nil, fmt.Errorf("building request: %w", err)
return nil, nil, errors.Wrap(err, "building request")
}
resp, err := s.client.Do(ctx, req, &alerts)
if err != nil {
return nil, resp, fmt.Errorf("performing request: %w", err)
return nil, resp, errors.Wrap(err, "performing request")
}
return &alerts, resp, nil
}

7
pkg/apiclient/alerts_service_test.go
View file

@ -8,13 +8,12 @@ import (
"reflect"
"testing"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/models"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestAlertsListAsMachine(t *testing.T) {

30
pkg/apiclient/auth.go
View file

@ -3,21 +3,23 @@ package apiclient
import (
"bytes"
"encoding/json"
"fmt"
"io"
"math/rand"
"net/http"
"net/http/httputil"
"net/url"
"sync"
"time"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
//"errors"
"fmt"
"io"
"net/http"
"net/http/httputil"
"net/url"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
//"google.golang.org/appengine/log"
)
type APIKeyTransport struct {
@ -167,11 +169,11 @@ func (t *JWTTransport) refreshJwtToken() error {
enc.SetEscapeHTML(false)
err = enc.Encode(auth)
if err != nil {
return fmt.Errorf("could not encode jwt auth body: %w", err)
return errors.Wrap(err, "could not encode jwt auth body")
}
req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s%s/watchers/login", t.URL, t.VersionPrefix), buf)
if err != nil {
return fmt.Errorf("could not create request: %w", err)
return errors.Wrap(err, "could not create request")
}
req.Header.Add("Content-Type", "application/json")
client := &http.Client{
@ -194,7 +196,7 @@ func (t *JWTTransport) refreshJwtToken() error {
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("could not get jwt token: %w", err)
return errors.Wrap(err, "could not get jwt token")
}
log.Debugf("auth-jwt : http %d", resp.StatusCode)
@ -215,10 +217,10 @@ func (t *JWTTransport) refreshJwtToken() error {
}
if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
return fmt.Errorf("unable to decode response: %w", err)
return errors.Wrap(err, "unable to decode response")
}
if err := t.Expiration.UnmarshalText([]byte(response.Expire)); err != nil {
return fmt.Errorf("unable to parse jwt expiration: %w", err)
return errors.Wrap(err, "unable to parse jwt expiration")
}
t.Token = response.Token
@ -261,7 +263,7 @@ func (t *JWTTransport) RoundTrip(req *http.Request) (*http.Response, error) {
if err != nil {
/*we had an error (network error for example, or 401 because token is refused), reset the token ?*/
t.Token = ""
return resp, fmt.Errorf("performing jwt auth: %w", err)
return resp, errors.Wrapf(err, "performing jwt auth")
}
log.Debugf("resp-jwt: %d", resp.StatusCode)

2
pkg/apiclient/auth_service.go
View file

@ -21,6 +21,7 @@ type enrollRequest struct {
}
func (s *AuthService) UnregisterWatcher(ctx context.Context) (*Response, error) {
u := fmt.Sprintf("%s/watchers", s.client.URLPrefix)
req, err := s.client.NewRequest(http.MethodDelete, u, nil)
if err != nil {
@ -35,6 +36,7 @@ func (s *AuthService) UnregisterWatcher(ctx context.Context) (*Response, error)
}
func (s *AuthService) RegisterWatcher(ctx context.Context, registration models.WatcherRegistrationRequest) (*Response, error) {
u := fmt.Sprintf("%s/watchers", s.client.URLPrefix)
req, err := s.client.NewRequest(http.MethodPost, u, &registration)

5
pkg/apiclient/auth_service_test.go
View file

@ -10,12 +10,11 @@ import (
"net/url"
"testing"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/models"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
)
type BasicMockPayload struct {

7
pkg/apiclient/client.go
View file

@ -11,6 +11,7 @@ import (
"net/url"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/pkg/errors"
)
var (
@ -124,9 +125,9 @@ func RegisterClient(config *Config, client *http.Client) (*ApiClient, error) {
/*if we have http status, return it*/
if err != nil {
if resp != nil && resp.Response != nil {
return nil, fmt.Errorf("api register (%s) http %s: %w", c.BaseURL, resp.Response.Status, err)
return nil, errors.Wrapf(err, "api register (%s) http %s : %s", c.BaseURL, resp.Response.Status, err)
}
return nil, fmt.Errorf("api register (%s): %w", c.BaseURL, err)
return nil, errors.Wrapf(err, "api register (%s) : %s", c.BaseURL, err)
}
return c, nil
@ -165,7 +166,7 @@ func CheckResponse(r *http.Response) error {
if err == nil && data != nil {
err := json.Unmarshal(data, errorResponse)
if err != nil {
return fmt.Errorf("http code %d, invalid body: %w", r.StatusCode, err)
return errors.Wrapf(err, "http code %d, invalid body", r.StatusCode)
}
} else {
errorResponse.Message = new(string)

4
pkg/apiclient/client_http_test.go
View file

@ -8,9 +8,9 @@ import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/stretchr/testify/assert"
)
func TestNewRequestInvalid(t *testing.T) {

3
pkg/apiclient/client_test.go
View file

@ -9,9 +9,10 @@ import (
"runtime"
"testing"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
)

7
pkg/apiclient/decisions_service.go
View file

@ -6,15 +6,14 @@ import (
"fmt"
"net/http"
qs "github.com/google/go-querystring/query"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/pkg/ptr"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/modelscapi"
"github.com/crowdsecurity/crowdsec/pkg/types"
qs "github.com/google/go-querystring/query"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
type DecisionsService service

7
pkg/apiclient/decisions_service_test.go
View file

@ -8,15 +8,14 @@ import (
"reflect"
"testing"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/pkg/ptr"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/modelscapi"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDecisionsList(t *testing.T) {

8
pkg/apiclient/decisions_sync_service.go
View file

@ -5,9 +5,9 @@ import (
"fmt"
"net/http"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
type DecisionDeleteService service
@ -18,12 +18,12 @@ func (d *DecisionDeleteService) Add(ctx context.Context, deletedDecisions *model
u := fmt.Sprintf("%s/decisions/delete", d.client.URLPrefix)
req, err := d.client.NewRequest(http.MethodPost, u, &deletedDecisions)
if err != nil {
return nil, nil, fmt.Errorf("while building request: %w", err)
return nil, nil, errors.Wrap(err, "while building request")
}
resp, err := d.client.Do(ctx, req, &response)
if err != nil {
return nil, resp, fmt.Errorf("while performing request: %w", err)
return nil, resp, errors.Wrap(err, "while performing request")
}
if resp.Response.StatusCode != http.StatusOK {
log.Warnf("Decisions delete response : http %s", resp.Response.Status)

5
pkg/apiclient/signal.go
View file

@ -8,6 +8,7 @@ import (
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/pkg/errors"
)
type SignalService service
@ -18,12 +19,12 @@ func (s *SignalService) Add(ctx context.Context, signals *models.AddSignalsReque
u := fmt.Sprintf("%s/signals", s.client.URLPrefix)
req, err := s.client.NewRequest(http.MethodPost, u, &signals)
if err != nil {
return nil, nil, fmt.Errorf("while building request: %w", err)
return nil, nil, errors.Wrap(err, "while building request")
}
resp, err := s.client.Do(ctx, req, &response)
if err != nil {
return nil, resp, fmt.Errorf("while performing request: %w", err)
return nil, resp, errors.Wrap(err, "while performing request")
}
if resp.Response.StatusCode != http.StatusOK {
log.Warnf("Signal push response : http %s", resp.Response.Status)

84
pkg/apiserver/apic.go
View file

@ -33,8 +33,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/types"
)
const (
// delta values must be smaller than the interval
var (
pullIntervalDefault = time.Hour * 2
pullIntervalDelta = 5 * time.Minute
pushIntervalDefault = time.Second * 10
@ -72,12 +71,7 @@ type apic struct {
// randomDuration returns a duration value between d-delta and d+delta
func randomDuration(d time.Duration, delta time.Duration) time.Duration {
ret := d + time.Duration(rand.Int63n(int64(2*delta))) - delta
// ticker interval must be > 0 (nanoseconds)
if ret <= 0 {
return 1
}
return ret
return time.Duration(float64(d) + float64(delta)*(-1.0+2.0*rand.Float64()))
}
func (a *apic) FetchScenariosListFromDB() ([]string, error) {
@ -828,6 +822,80 @@ func (a *apic) Pull() error {
}
}
func (a *apic) GetMetrics() (*models.Metrics, error) {
metric := &models.Metrics{
ApilVersion: ptr.Of(version.String()),
Machines: make([]*models.MetricsAgentInfo, 0),
Bouncers: make([]*models.MetricsBouncerInfo, 0),
}
machines, err := a.dbClient.ListMachines()
if err != nil {
return metric, err
}
bouncers, err := a.dbClient.ListBouncers()
if err != nil {
return metric, err
}
var lastpush string
for _, machine := range machines {
if machine.LastPush == nil {
lastpush = time.Time{}.String()
} else {
lastpush = machine.LastPush.String()
}
m := &models.MetricsAgentInfo{
Version: machine.Version,
Name: machine.MachineId,
LastUpdate: machine.UpdatedAt.String(),
LastPush: lastpush,
}
metric.Machines = append(metric.Machines, m)
}
for _, bouncer := range bouncers {
m := &models.MetricsBouncerInfo{
Version: bouncer.Version,
CustomName: bouncer.Name,
Name: bouncer.Type,
LastPull: bouncer.LastPull.String(),
}
metric.Bouncers = append(metric.Bouncers, m)
}
return metric, nil
}
func (a *apic) SendMetrics(stop chan (bool)) {
defer trace.CatchPanic("lapi/metricsToAPIC")
ticker := time.NewTicker(a.metricsIntervalFirst)
log.Infof("Start send metrics to CrowdSec Central API (interval: %s once, then %s)", a.metricsIntervalFirst.Round(time.Second), a.metricsInterval)
for {
metrics, err := a.GetMetrics()
if err != nil {
log.Errorf("unable to get metrics (%s), will retry", err)
}
_, _, err = a.apiClient.Metrics.Add(context.Background(), metrics)
if err != nil {
log.Errorf("capi metrics: failed: %s", err)
} else {
log.Infof("capi metrics: metrics sent successfully")
}
select {
case <-stop:
return
case <-ticker.C:
ticker.Reset(a.metricsInterval)
case <-a.metricsTomb.Dying(): // if one apic routine is dying, do we kill the others?
a.pullTomb.Kill(nil)
a.pushTomb.Kill(nil)
return
}
}
}
func (a *apic) Shutdown() {
a.pushTomb.Kill(nil)
a.pullTomb.Kill(nil)

145
pkg/apiserver/apic_metrics.go
View file

@ -1,145 +0,0 @@
package apiserver
import (
"context"
"time"
log "github.com/sirupsen/logrus"
"golang.org/x/exp/slices"
"github.com/crowdsecurity/go-cs-lib/pkg/ptr"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/models"
)
func (a *apic) GetMetrics() (*models.Metrics, error) {
machines, err := a.dbClient.ListMachines()
if err != nil {
return nil, err
}
machinesInfo := make([]*models.MetricsAgentInfo, len(machines))
for i, machine := range machines {
machinesInfo[i] = &models.MetricsAgentInfo{
Version: machine.Version,
Name: machine.MachineId,
LastUpdate: machine.UpdatedAt.String(),
LastPush: ptr.OrEmpty(machine.LastPush).String(),
}
}
bouncers, err := a.dbClient.ListBouncers()
if err != nil {
return nil, err
}
bouncersInfo := make([]*models.MetricsBouncerInfo, len(bouncers))
for i, bouncer := range bouncers {
bouncersInfo[i] = &models.MetricsBouncerInfo{
Version: bouncer.Version,
CustomName: bouncer.Name,
Name: bouncer.Type,
LastPull: bouncer.LastPull.String(),
}
}
return &models.Metrics{
ApilVersion: ptr.Of(version.String()),
Machines: machinesInfo,
Bouncers: bouncersInfo,
}, nil
}
func (a *apic) fetchMachineIDs() ([]string, error) {
machines, err := a.dbClient.ListMachines()
if err != nil {
return nil, err
}
ret := make([]string, len(machines))
for i, machine := range machines {
ret[i] = machine.MachineId
}
// sorted slices are required for the slices.Equal comparison
slices.Sort(ret)
return ret, nil
}
// SendMetrics sends metrics to the API server until it receives a stop signal.
//
// Metrics are sent at start, then at the randomized metricsIntervalFirst,
// then at regular metricsInterval. If a change is detected in the list
// of machines, the next metrics are sent immediately.
func (a *apic) SendMetrics(stop chan (bool)) {
defer trace.CatchPanic("lapi/metricsToAPIC")
// verify the list of machines every <checkInt> interval
const checkInt = 20 * time.Second
// intervals must always be > 0
metInts := []time.Duration{1, a.metricsIntervalFirst, a.metricsInterval}
log.Infof("Start send metrics to CrowdSec Central API (interval: %s once, then %s)",
metInts[1].Round(time.Second), metInts[2])
count := -1
nextMetInt := func() time.Duration {
if count < len(metInts)-1 {
count++
}
return metInts[count]
}
// store the list of machine IDs to compare
// with the next list
machineIDs := []string{}
reloadMachineIDs := func() {
ids, err := a.fetchMachineIDs()
if err != nil {
log.Debugf("unable to get machines (%s), will retry", err)
return
}
machineIDs = ids
}
checkTicker := time.NewTicker(checkInt)
metTicker := time.NewTicker(nextMetInt())
for {
select {
case <-stop:
checkTicker.Stop()
metTicker.Stop()
return
case <-checkTicker.C:
oldIDs := machineIDs
reloadMachineIDs()
if !slices.Equal(oldIDs, machineIDs) {
log.Infof("capi metrics: machines changed, immediate send")
metTicker.Reset(1)
}
case <-metTicker.C:
metrics, err := a.GetMetrics()
if err != nil {
log.Errorf("unable to get metrics (%s), will retry", err)
}
log.Info("capi metrics: sending")
_, _, err = a.apiClient.Metrics.Add(context.Background(), metrics)
if err != nil {
log.Errorf("capi metrics: failed: %s", err)
}
metTicker.Reset(nextMetInt())
case <-a.metricsTomb.Dying(): // if one apic routine is dying, do we kill the others?
checkTicker.Stop()
metTicker.Stop()
a.pullTomb.Kill(nil)
a.pushTomb.Kill(nil)
return
}
}
}

101
pkg/apiserver/apic_metrics_test.go
View file

@ -1,101 +0,0 @@
package apiserver
import (
"context"
"fmt"
"net/url"
"testing"
"time"
"github.com/jarcoal/httpmock"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
)
func TestAPICSendMetrics(t *testing.T) {
tests := []struct {
name string
duration time.Duration
expectedCalls int
setUp func(*apic)
metricsInterval time.Duration
}{
{
name: "basic",
duration: time.Millisecond * 30,
metricsInterval: time.Millisecond * 5,
expectedCalls: 5,
setUp: func(api *apic) {},
},
{
name: "with some metrics",
duration: time.Millisecond * 30,
metricsInterval: time.Millisecond * 5,
expectedCalls: 5,
setUp: func(api *apic) {
api.dbClient.Ent.Machine.Delete().ExecX(context.Background())
api.dbClient.Ent.Machine.Create().
SetMachineId("1234").
SetPassword(testPassword.String()).
SetIpAddress("1.2.3.4").
SetScenarios("crowdsecurity/test").
SetLastPush(time.Time{}).
SetUpdatedAt(time.Time{}).
ExecX(context.Background())
api.dbClient.Ent.Bouncer.Delete().ExecX(context.Background())
api.dbClient.Ent.Bouncer.Create().
SetIPAddress("1.2.3.6").
SetName("someBouncer").
SetAPIKey("foobar").
SetRevoked(false).
SetLastPull(time.Time{}).
ExecX(context.Background())
},
},
}
httpmock.RegisterResponder("POST", "http://api.crowdsec.net/api/metrics/", httpmock.NewBytesResponder(200, []byte{}))
httpmock.Activate()
defer httpmock.Deactivate()
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
url, err := url.ParseRequestURI("http://api.crowdsec.net/")
require.NoError(t, err)
apiClient, err := apiclient.NewDefaultClient(
url,
"/api",
fmt.Sprintf("crowdsec/%s", version.String()),
nil,
)
require.NoError(t, err)
api := getAPIC(t)
api.pushInterval = time.Millisecond
api.pushIntervalFirst = time.Millisecond
api.apiClient = apiClient
api.metricsInterval = tc.metricsInterval
api.metricsIntervalFirst = tc.metricsInterval
tc.setUp(api)
stop := make(chan bool)
httpmock.ZeroCallCounters()
go api.SendMetrics(stop)
time.Sleep(tc.duration)
stop <- true
info := httpmock.GetCallCountInfo()
noResponderCalls := info["NO_RESPONDER"]
responderCalls := info["POST http://api.crowdsec.net/api/metrics/"]
assert.LessOrEqual(t, absDiff(tc.expectedCalls, responderCalls), 2)
assert.Zero(t, noResponderCalls)
})
}
}

84
pkg/apiserver/apic_test.go
View file

@ -1057,6 +1057,90 @@ func TestAPICPush(t *testing.T) {
}
}
func TestAPICSendMetrics(t *testing.T) {
tests := []struct {
name string
duration time.Duration
expectedCalls int
setUp func(*apic)
metricsInterval time.Duration
}{
{
name: "basic",
duration: time.Millisecond * 30,
metricsInterval: time.Millisecond * 5,
expectedCalls: 5,
setUp: func(api *apic) {},
},
{
name: "with some metrics",
duration: time.Millisecond * 30,
metricsInterval: time.Millisecond * 5,
expectedCalls: 5,
setUp: func(api *apic) {
api.dbClient.Ent.Machine.Delete().ExecX(context.Background())
api.dbClient.Ent.Machine.Create().
SetMachineId("1234").
SetPassword(testPassword.String()).
SetIpAddress("1.2.3.4").
SetScenarios("crowdsecurity/test").
SetLastPush(time.Time{}).
SetUpdatedAt(time.Time{}).
ExecX(context.Background())
api.dbClient.Ent.Bouncer.Delete().ExecX(context.Background())
api.dbClient.Ent.Bouncer.Create().
SetIPAddress("1.2.3.6").
SetName("someBouncer").
SetAPIKey("foobar").
SetRevoked(false).
SetLastPull(time.Time{}).
ExecX(context.Background())
},
},
}
httpmock.RegisterResponder("POST", "http://api.crowdsec.net/api/metrics/", httpmock.NewBytesResponder(200, []byte{}))
httpmock.Activate()
defer httpmock.Deactivate()
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
url, err := url.ParseRequestURI("http://api.crowdsec.net/")
require.NoError(t, err)
apiClient, err := apiclient.NewDefaultClient(
url,
"/api",
fmt.Sprintf("crowdsec/%s", version.String()),
nil,
)
require.NoError(t, err)
api := getAPIC(t)
api.pushInterval = time.Millisecond
api.pushIntervalFirst = time.Millisecond
api.apiClient = apiClient
api.metricsInterval = tc.metricsInterval
api.metricsIntervalFirst = tc.metricsInterval
tc.setUp(api)
stop := make(chan bool)
httpmock.ZeroCallCounters()
go api.SendMetrics(stop)
time.Sleep(tc.duration)
stop <- true
info := httpmock.GetCallCountInfo()
noResponderCalls := info["NO_RESPONDER"]
responderCalls := info["POST http://api.crowdsec.net/api/metrics/"]
assert.LessOrEqual(t, absDiff(tc.expectedCalls, responderCalls), 2)
assert.Zero(t, noResponderCalls)
})
}
}
func TestAPICPull(t *testing.T) {
api := getAPIC(t)
tests := []struct {

28
pkg/apiserver/apiserver.go
View file

@ -9,18 +9,9 @@ import (
"net"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/gin-gonic/gin"
"github.com/go-co-op/gocron"
"github.com/golang-jwt/jwt/v4"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"gopkg.in/natefinch/lumberjack.v2"
"gopkg.in/tomb.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
@ -31,6 +22,13 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/gin-gonic/gin"
"github.com/go-co-op/gocron"
"github.com/golang-jwt/jwt/v4"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"gopkg.in/natefinch/lumberjack.v2"
"gopkg.in/tomb.v2"
)
var (
@ -118,7 +116,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) {
logFile := ""
if config.LogMedia == "file" {
logFile = filepath.Join(config.LogDir, "crowdsec_api.log")
logFile = fmt.Sprintf("%s/crowdsec_api.log", config.LogDir)
}
if log.GetLevel() < log.DebugLevel {
@ -164,7 +162,15 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) {
if config.CompressLogs != nil {
_compress = *config.CompressLogs
}
/*cf. https://github.com/natefinch/lumberjack/issues/82
let's create the file beforehand w/ the right perms */
// check if file exists
_, err := os.Stat(logFile)
// create file if not exists, purposefully ignore errors
if os.IsNotExist(err) {
file, _ := os.OpenFile(logFile, os.O_RDWR|os.O_CREATE, 0600)
file.Close()
}
LogOutput := &lumberjack.Logger{
Filename: logFile,
MaxSize: _maxsize, //megabytes

6
pkg/apiserver/controllers/v1/controller.go
View file

@ -2,15 +2,17 @@ package v1
import (
"context"
"fmt"
"net"
//"github.com/crowdsecurity/crowdsec/pkg/apiserver/controllers"
middlewares "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/csplugin"
"github.com/crowdsecurity/crowdsec/pkg/csprofiles"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/pkg/errors"
)
type Controller struct {
@ -46,7 +48,7 @@ func New(cfg *ControllerV1Config) (*Controller, error) {
profiles, err := csprofiles.NewProfile(cfg.ProfilesCfg)
if err != nil {
return &Controller{}, fmt.Errorf("failed to compile profiles: %w", err)
return &Controller{}, errors.Wrapf(err, "failed to compile profiles")
}
v1 := &Controller{

10
pkg/apiserver/middlewares/v1/api_key.go
View file

@ -3,7 +3,7 @@ package v1
import (
"crypto/rand"
"crypto/sha512"
"encoding/base64"
"encoding/hex"
"fmt"
"net/http"
"strings"
@ -15,11 +15,9 @@ import (
log "github.com/sirupsen/logrus"
)
const (
var (
APIKeyHeader = "X-Api-Key"
bouncerContextKey = "bouncer_info"
// max allowed by bcrypt 72 = 54 bytes in base64
dummyAPIKeySize = 54
)
type APIKey struct {
@ -33,7 +31,7 @@ func GenerateAPIKey(n int) (string, error) {
if _, err := rand.Read(bytes); err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(bytes), nil
return hex.EncodeToString(bytes), nil
}
func NewAPIKey(dbClient *database.Client) *APIKey {
@ -84,7 +82,7 @@ func (a *APIKey) MiddlewareFunc() gin.HandlerFunc {
if err != nil && strings.Contains(err.Error(), "bouncer not found") {
//Because we have a valid cert, automatically create the bouncer in the database if it does not exist
//Set a random API key, but it will never be used
apiKey, err := GenerateAPIKey(dummyAPIKeySize)
apiKey, err := GenerateAPIKey(64)
if err != nil {
log.WithFields(log.Fields{
"ip": c.ClientIP(),

2
pkg/apiserver/middlewares/v1/jwt.go
View file

@ -81,7 +81,7 @@ func (j *JWT) Authenticator(c *gin.Context) (interface{}, error) {
//Machine was not found, let's create it
log.Printf("machine %s not found, create it", machineID)
//let's use an apikey as the password, doesn't matter in this case (generatePassword is only available in cscli)
pwd, err := GenerateAPIKey(dummyAPIKeySize)
pwd, err := GenerateAPIKey(64)
if err != nil {
log.WithFields(log.Fields{
"ip": c.ClientIP(),

7
pkg/apiserver/middlewares/v1/tls_auth.go
View file

@ -12,6 +12,7 @@ import (
"time"
"github.com/gin-gonic/gin"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"golang.org/x/crypto/ocsp"
)
@ -175,9 +176,9 @@ func (ta *TLSAuth) isInvalid(cert *x509.Certificate, issuer *x509.Certificate) (
}
revoked, err := ta.isRevoked(cert, issuer)
if err != nil {
//Fail securely, if we can't check the revocation status, let's consider the cert invalid
//Fail securely, if we can't check the revokation status, let's consider the cert invalid
//We may change this in the future based on users feedback, but this seems the most sensible thing to do
return true, fmt.Errorf("could not check for client certification revocation status: %w", err)
return true, errors.Wrap(err, "could not check for client certification revokation status")
}
return revoked, nil
@ -230,7 +231,7 @@ func (ta *TLSAuth) ValidateCert(c *gin.Context) (bool, string, error) {
revoked, err := ta.isInvalid(clientCert, c.Request.TLS.VerifiedChains[0][1])
if err != nil {
ta.logger.Errorf("TLSAuth: error checking if client certificate is revoked: %s", err)
return false, "", fmt.Errorf("could not check for client certification revokation status: %w", err)
return false, "", errors.Wrap(err, "could not check for client certification revokation status")
}
if revoked {
return false, "", fmt.Errorf("client certificate is revoked")

3
pkg/csconfig/common.go
View file

@ -4,6 +4,7 @@ import (
"fmt"
"path/filepath"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
@ -38,7 +39,7 @@ func (c *Config) LoadCommon() error {
}
*k, err = filepath.Abs(*k)
if err != nil {
return fmt.Errorf("failed to get absolute path of '%s': %w", *k, err)
return errors.Wrapf(err, "failed to get absolute path of '%s'", *k)
}
}

4
pkg/csconfig/config_paths.go
View file

@ -3,6 +3,8 @@ package csconfig
import (
"fmt"
"path/filepath"
"github.com/pkg/errors"
)
type ConfigurationPaths struct {
@ -48,7 +50,7 @@ func (c *Config) LoadConfigurationPaths() error {
}
*k, err = filepath.Abs(*k)
if err != nil {
return fmt.Errorf("failed to get absolute path of '%s': %w", *k, err)
return errors.Wrapf(err, "failed to get absolute path of '%s'", *k)
}
}

4
pkg/csconfig/profiles.go
View file

@ -2,13 +2,13 @@ package csconfig
import (
"bytes"
"errors"
"fmt"
"io"
"github.com/crowdsecurity/go-cs-lib/pkg/yamlpatch"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/pkg/errors"
"gopkg.in/yaml.v2"
)
@ -53,7 +53,7 @@ func (c *LocalApiServerCfg) LoadProfiles() error {
if errors.Is(err, io.EOF) {
break
}
return fmt.Errorf("while decoding %s: %w", c.ProfilesPath, err)
return errors.Wrapf(err, "while decoding %s", c.ProfilesPath)
}
c.Profiles = append(c.Profiles, &t)
}

12
pkg/csplugin/broker.go
View file

@ -2,7 +2,6 @@ package csplugin
import (
"context"
"errors"
"fmt"
"io"
"os"
@ -15,6 +14,7 @@ import (
"github.com/Masterminds/sprig/v3"
"github.com/google/uuid"
plugin "github.com/hashicorp/go-plugin"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
@ -83,10 +83,10 @@ func (pb *PluginBroker) Init(pluginCfg *csconfig.PluginCfg, profileConfigs []*cs
pb.pluginProcConfig = pluginCfg
pb.pluginsTypesToDispatch = make(map[string]struct{})
if err := pb.loadConfig(configPaths.NotificationDir); err != nil {
return fmt.Errorf("while loading plugin config: %w", err)
return errors.Wrap(err, "while loading plugin config")
}
if err := pb.loadPlugins(configPaths.PluginDir); err != nil {
return fmt.Errorf("while loading plugin: %w", err)
return errors.Wrap(err, "while loading plugin")
}
pb.watcher = PluginWatcher{}
pb.watcher.Init(pb.pluginConfigByName, pb.alertsByPluginName)
@ -268,7 +268,7 @@ func (pb *PluginBroker) loadPlugins(path string) error {
data = []byte(csstring.StrictExpand(string(data), os.LookupEnv))
_, err = pluginClient.Configure(context.Background(), &protobufs.Config{Config: data})
if err != nil {
return fmt.Errorf("while configuring %s: %w", pc.Name, err)
return errors.Wrapf(err, "while configuring %s", pc.Name)
}
log.Infof("registered plugin %s", pc.Name)
pb.notificationPluginByName[pc.Name] = pluginClient
@ -354,7 +354,7 @@ func ParsePluginConfigFile(path string) ([]PluginConfig, error) {
parsedConfigs := make([]PluginConfig, 0)
yamlFile, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("while opening %s: %w", path, err)
return parsedConfigs, errors.Wrapf(err, "while opening %s", path)
}
dec := yaml.NewDecoder(yamlFile)
dec.SetStrict(true)
@ -365,7 +365,7 @@ func ParsePluginConfigFile(path string) ([]PluginConfig, error) {
if errors.Is(err, io.EOF) {
break
}
return nil, fmt.Errorf("while decoding %s got error %s", path, err)
return []PluginConfig{}, fmt.Errorf("while decoding %s got error %s", path, err)
}
// if the yaml document is empty, skip
if reflect.DeepEqual(pc, PluginConfig{}) {

18
pkg/csplugin/broker_suite_test.go
View file

@ -14,6 +14,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
type PluginSuite struct {
suite.Suite
@ -22,19 +23,21 @@ type PluginSuite struct {
// full path to the built plugin binary
builtBinary string
runDir string // temporary directory for each test
pluginDir string // (config_paths.plugin_dir)
notifDir string // (config_paths.notification_dir)
pluginBinary string // full path to the plugin binary (unique for each test)
pluginConfig string // full path to the notification config (unique for each test)
runDir string // temporary directory for each test
pluginDir string // (config_paths.plugin_dir)
notifDir string // (config_paths.notification_dir)
pluginBinary string // full path to the plugin binary (unique for each test)
pluginConfig string // full path to the notification config (unique for each test)
pluginBroker *PluginBroker
}
func TestPluginSuite(t *testing.T) {
suite.Run(t, new(PluginSuite))
}
func (s *PluginSuite) SetupSuite() {
var err error
@ -54,12 +57,14 @@ func (s *PluginSuite) SetupSuite() {
require.NoError(t, err, "while building dummy plugin")
}
func (s *PluginSuite) TearDownSuite() {
t := s.T()
err := os.RemoveAll(s.buildDir)
require.NoError(t, err)
}
func copyFile(src string, dst string) error {
s, err := os.Open(src)
if err != nil {
@ -94,6 +99,7 @@ func (s *PluginSuite) TearDownTest() {
s.TearDownSubTest()
}
func (s *PluginSuite) SetupSubTest() {
var err error
t := s.T()
@ -119,7 +125,7 @@ func (s *PluginSuite) SetupSubTest() {
require.NoError(t, err, "while copying built binary")
err = os.Chmod(s.pluginBinary, 0o744)
require.NoError(t, err, "chmod 0744 %s", s.pluginBinary)
s.pluginConfig = path.Join(s.notifDir, "dummy.yaml")
err = copyFile("testdata/dummy.yaml", s.pluginConfig)
require.NoError(t, err, "while copying plugin config")

19
pkg/csplugin/broker_test.go
View file

@ -22,6 +22,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/models"
)
func (s *PluginSuite) permissionSetter(perm os.FileMode) func(*testing.T) {
return func(t *testing.T) {
err := os.Chmod(s.pluginBinary, perm)
@ -29,28 +30,30 @@ func (s *PluginSuite) permissionSetter(perm os.FileMode) func(*testing.T) {
}
}
func (s *PluginSuite) readconfig() PluginConfig {
func (s *PluginSuite) readconfig() (PluginConfig) {
var config PluginConfig
t := s.T()
orig, err := os.ReadFile(s.pluginConfig)
require.NoError(t, err, "unable to read config file %s", s.pluginConfig)
require.NoError(t, err,"unable to read config file %s", s.pluginConfig)
err = yaml.Unmarshal(orig, &config)
require.NoError(t, err, "unable to unmarshal config file")
require.NoError(t, err,"unable to unmarshal config file")
return config
}
func (s *PluginSuite) writeconfig(config PluginConfig) {
t := s.T()
data, err := yaml.Marshal(&config)
require.NoError(t, err, "unable to marshal config file")
require.NoError(t, err,"unable to marshal config file")
err = os.WriteFile(s.pluginConfig, data, 0644)
require.NoError(t, err, "unable to write config file %s", s.pluginConfig)
require.NoError(t, err,"unable to write config file %s", s.pluginConfig)
}
func (s *PluginSuite) TestBrokerInit() {
tests := []struct {
name string
@ -59,7 +62,7 @@ func (s *PluginSuite) TestBrokerInit() {
expectedErr string
}{
{
name: "valid config",
name: "valid config",
},
{
name: "group writable binary",
@ -346,7 +349,7 @@ func (s *PluginSuite) TestBrokerRunSimple() {
DefaultEmptyTicker = 50 * time.Millisecond
t := s.T()
pb, err := s.InitBroker(nil)
assert.NoError(t, err)

2
pkg/csplugin/broker_win_test.go
View file

@ -33,7 +33,7 @@ func (s *PluginSuite) TestBrokerInit() {
expectedErr string
}{
{
name: "valid config",
name: "valid config",
},
{
name: "no plugin dir",

3
pkg/csplugin/listfiles.go
View file

@ -13,9 +13,10 @@ func listFilesAtPath(path string) ([]string, error) {
return nil, err
}
for _, file := range files {
if !file.IsDir() {
if ! file.IsDir() {
filePaths = append(filePaths, filepath.Join(path, file.Name()))
}
}
return filePaths, nil
}

10
pkg/csplugin/listfiles_test.go
View file

@ -27,9 +27,9 @@ func TestListFilesAtPath(t *testing.T) {
require.NoError(t, err)
tests := []struct {
name string
path string
want []string
name string
path string
want []string
expectedErr string
}{
{
@ -41,8 +41,8 @@ func TestListFilesAtPath(t *testing.T) {
},
},
{
name: "invalid directory",
path: "./foo/bar/",
name: "invalid directory",
path: "./foo/bar/",
expectedErr: "open ./foo/bar/: " + cstest.PathNotFoundMessage,
},
}

3
pkg/csplugin/notifier.go
View file

@ -4,10 +4,9 @@ import (
"context"
"fmt"
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
plugin "github.com/hashicorp/go-plugin"
"google.golang.org/grpc"
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
)
type Notifier interface {

11
pkg/csplugin/utils.go
View file

@ -3,7 +3,6 @@
package csplugin
import (
"errors"
"fmt"
"io/fs"
"math"
@ -14,6 +13,8 @@ import (
"strconv"
"strings"
"syscall"
"github.com/pkg/errors"
)
func CheckCredential(uid int, gid int) *syscall.SysProcAttr {
@ -34,7 +35,7 @@ func (pb *PluginBroker) CreateCmd(binaryPath string) (*exec.Cmd, error) {
}
cmd.SysProcAttr, err = getProcessAttr(pb.pluginProcConfig.User, pb.pluginProcConfig.Group)
if err != nil {
return nil, fmt.Errorf("while getting process attributes: %w", err)
return nil, errors.Wrap(err, "while getting process attributes")
}
cmd.SysProcAttr.Credential.NoSetGroups = true
}
@ -104,17 +105,17 @@ func pluginIsValid(path string) error {
// check if it exists
if details, err = os.Stat(path); err != nil {
return fmt.Errorf("plugin at %s does not exist: %w", path, err)
return errors.Wrap(err, fmt.Sprintf("plugin at %s does not exist", path))
}
// check if it is owned by current user
currentUser, err := user.Current()
if err != nil {
return fmt.Errorf("while getting current user: %w", err)
return errors.Wrap(err, "while getting current user")
}
currentUID, err := getUID(currentUser.Username)
if err != nil {
return fmt.Errorf("while looking up the current uid: %w", err)
return errors.Wrap(err, "while looking up the current uid")
}
stat := details.Sys().(*syscall.Stat_t)
if stat.Uid != currentUID {

35
pkg/csplugin/utils_windows.go
View file

@ -13,6 +13,7 @@ import (
"syscall"
"unsafe"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"golang.org/x/sys/windows"
)
@ -53,38 +54,38 @@ func CheckPerms(path string) error {
systemSid, err := windows.CreateWellKnownSid(windows.WELL_KNOWN_SID_TYPE(windows.WinLocalSystemSid))
if err != nil {
return fmt.Errorf("while creating SYSTEM well known sid: %w", err)
return errors.Wrap(err, "while creating SYSTEM well known sid")
}
adminSid, err := windows.CreateWellKnownSid(windows.WELL_KNOWN_SID_TYPE(windows.WinBuiltinAdministratorsSid))
if err != nil {
return fmt.Errorf("while creating built-in Administrators well known sid: %w", err)
return errors.Wrap(err, "while creating built-in Administrators well known sid")
}
currentUser, err := user.Current()
if err != nil {
return fmt.Errorf("while getting current user: %w", err)
return errors.Wrap(err, "while getting current user")
}
currentUserSid, _, _, err := windows.LookupSID("", currentUser.Username)
if err != nil {
return fmt.Errorf("while looking up current user sid: %w", err)
return errors.Wrap(err, "while looking up current user sid")
}
sd, err := windows.GetNamedSecurityInfo(path, windows.SE_FILE_OBJECT, windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION)
if err != nil {
return fmt.Errorf("while getting owner security info: %w", err)
return errors.Wrap(err, "while getting owner security info")
}
if !sd.IsValid() {
return fmt.Errorf("security descriptor is invalid")
return errors.New("security descriptor is invalid")
}
owner, _, err := sd.Owner()
if err != nil {
return fmt.Errorf("while getting owner: %w", err)
return errors.Wrap(err, "while getting owner")
}
if !owner.IsValid() {
return fmt.Errorf("owner is invalid")
return errors.New("owner is invalid")
}
if !owner.Equals(systemSid) && !owner.Equals(currentUserSid) && !owner.Equals(adminSid) {
@ -93,7 +94,7 @@ func CheckPerms(path string) error {
dacl, _, err := sd.DACL()
if err != nil {
return fmt.Errorf("while getting DACL: %w", err)
return errors.Wrap(err, "while getting DACL")
}
if dacl == nil {
@ -101,7 +102,7 @@ func CheckPerms(path string) error {
}
if err != nil {
return fmt.Errorf("while looking up current user sid: %w", err)
return errors.Wrap(err, "while looking up current user sid")
}
rs := reflect.ValueOf(dacl).Elem()
@ -123,7 +124,7 @@ func CheckPerms(path string) error {
ace := &AccessAllowedAce{}
ret, _, _ := procGetAce.Call(uintptr(unsafe.Pointer(dacl)), uintptr(i), uintptr(unsafe.Pointer(&ace)))
if ret == 0 {
return fmt.Errorf("while getting ACE: %w", windows.GetLastError())
return errors.Wrap(windows.GetLastError(), "while getting ACE")
}
log.Debugf("ACE %d: %+v\n", i, ace)
@ -161,14 +162,14 @@ func getProcessAtr() (*syscall.SysProcAttr, error) {
err := windows.OpenProcessToken(proc, windows.TOKEN_DUPLICATE|windows.TOKEN_ADJUST_DEFAULT|
windows.TOKEN_QUERY|windows.TOKEN_ASSIGN_PRIMARY|windows.TOKEN_ADJUST_GROUPS|windows.TOKEN_ADJUST_PRIVILEGES, &procToken)
if err != nil {
return nil, fmt.Errorf("while opening process token: %w", err)
return nil, errors.Wrapf(err, "while opening process token")
}
defer procToken.Close()
err = windows.DuplicateTokenEx(procToken, 0, nil, windows.SecurityImpersonation,
windows.TokenPrimary, &token)
if err != nil {
return nil, fmt.Errorf("while duplicating token: %w", err)
return nil, errors.Wrapf(err, "while duplicating token")
}
//Remove all privileges from the token
@ -176,7 +177,7 @@ func getProcessAtr() (*syscall.SysProcAttr, error) {
err = windows.AdjustTokenPrivileges(token, true, nil, 0, nil, nil)
if err != nil {
return nil, fmt.Errorf("while adjusting token privileges: %w", err)
return nil, errors.Wrapf(err, "while adjusting token privileges")
}
//Run the plugin as a medium integrity level process
@ -194,7 +195,7 @@ func getProcessAtr() (*syscall.SysProcAttr, error) {
(*byte)(unsafe.Pointer(tml)), tml.Size())
if err != nil {
token.Close()
return nil, fmt.Errorf("while setting token information: %w", err)
return nil, errors.Wrapf(err, "while setting token information")
}
return &windows.SysProcAttr{
@ -208,7 +209,7 @@ func (pb *PluginBroker) CreateCmd(binaryPath string) (*exec.Cmd, error) {
cmd := exec.Command(binaryPath)
cmd.SysProcAttr, err = getProcessAtr()
if err != nil {
return nil, fmt.Errorf("while getting process attributes: %w", err)
return nil, errors.Wrap(err, "while getting process attributes")
}
return cmd, err
}
@ -228,7 +229,7 @@ func pluginIsValid(path string) error {
// check if it exists
if _, err = os.Stat(path); err != nil {
return fmt.Errorf("plugin at %s does not exist", path)
return errors.Wrap(err, fmt.Sprintf("plugin at %s does not exist", path))
}
// check if it is owned by root

3
pkg/csplugin/watcher.go
View file

@ -4,10 +4,9 @@ import (
"sync"
"time"
"github.com/crowdsecurity/crowdsec/pkg/models"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
"github.com/crowdsecurity/crowdsec/pkg/models"
)
/*

17
pkg/csplugin/watcher_test.go
View file

@ -7,12 +7,9 @@ import (
"testing"
"time"
"github.com/stretchr/testify/require"
"gopkg.in/tomb.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/cstest"
"github.com/crowdsecurity/crowdsec/pkg/models"
"gopkg.in/tomb.v2"
"gotest.tools/v3/assert"
)
var ctx = context.Background()
@ -67,7 +64,7 @@ func TestPluginWatcherInterval(t *testing.T) {
ct, cancel := context.WithTimeout(ctx, time.Microsecond)
defer cancel()
err := listenChannelWithTimeout(ct, pw.PluginEvents)
cstest.RequireErrorContains(t, err, "context deadline exceeded")
assert.ErrorContains(t, err, "context deadline exceeded")
resetTestTomb(&testTomb, &pw)
testTomb = tomb.Tomb{}
pw.Start(&testTomb)
@ -75,7 +72,7 @@ func TestPluginWatcherInterval(t *testing.T) {
ct, cancel = context.WithTimeout(ctx, time.Millisecond*5)
defer cancel()
err = listenChannelWithTimeout(ct, pw.PluginEvents)
require.NoError(t, err)
assert.NilError(t, err)
resetTestTomb(&testTomb, &pw)
// This is to avoid the int complaining
}
@ -99,7 +96,7 @@ func TestPluginAlertCountWatcher(t *testing.T) {
ct, cancel := context.WithTimeout(ctx, time.Second)
defer cancel()
err := listenChannelWithTimeout(ct, pw.PluginEvents)
cstest.RequireErrorContains(t, err, "context deadline exceeded")
assert.ErrorContains(t, err, "context deadline exceeded")
// Channel won't contain any events since threshold is not crossed.
resetWatcherAlertCounter(&pw)
@ -107,7 +104,7 @@ func TestPluginAlertCountWatcher(t *testing.T) {
ct, cancel = context.WithTimeout(ctx, time.Second)
defer cancel()
err = listenChannelWithTimeout(ct, pw.PluginEvents)
cstest.RequireErrorContains(t, err, "context deadline exceeded")
assert.ErrorContains(t, err, "context deadline exceeded")
// Channel will contain an event since threshold is crossed.
resetWatcherAlertCounter(&pw)
@ -115,6 +112,6 @@ func TestPluginAlertCountWatcher(t *testing.T) {
ct, cancel = context.WithTimeout(ctx, time.Second)
defer cancel()
err = listenChannelWithTimeout(ct, pw.PluginEvents)
require.NoError(t, err)
assert.NilError(t, err)
resetTestTomb(&testTomb, &pw)
}

9
pkg/csprofiles/csprofiles.go
View file

@ -6,13 +6,12 @@ import (
"github.com/antonmedv/expr"
"github.com/antonmedv/expr/vm"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
type Runtime struct {
@ -48,10 +47,10 @@ func NewProfile(profilesCfg []*csconfig.ProfileCfg) ([]*Runtime, error) {
runtime.DebugFilters = make([]*exprhelpers.ExprDebugger, len(profile.Filters))
runtime.Cfg = profile
if runtime.Cfg.OnSuccess != "" && runtime.Cfg.OnSuccess != "continue" && runtime.Cfg.OnSuccess != "break" {
return []*Runtime{}, fmt.Errorf("invalid 'on_success' for '%s': %s", profile.Name, runtime.Cfg.OnSuccess)
return []*Runtime{}, errors.Wrapf(err, "invalid 'on_success' for '%s' : %s", profile.Name, runtime.Cfg.OnSuccess)
}
if runtime.Cfg.OnFailure != "" && runtime.Cfg.OnFailure != "continue" && runtime.Cfg.OnFailure != "break" && runtime.Cfg.OnFailure != "apply" {
return []*Runtime{}, fmt.Errorf("invalid 'on_failure' for '%s' : %s", profile.Name, runtime.Cfg.OnFailure)
return []*Runtime{}, errors.Wrapf(err, "invalid 'on_failure' for '%s' : %s", profile.Name, runtime.Cfg.OnFailure)
}
for fIdx, filter := range profile.Filters {

Some files were not shown because too many files have changed in this diff Show more