beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

docker: fix/improve support for persistent configurations (#1915)

Browse source 
set all defaults in config.yaml and leave environment variables empty. This way when they are set we know that we must override the values in config.yaml.
ignore tainted objects when calling install/upgrade/remove
use_wal is false by default
This commit is contained in:
mmetc 2022-12-10 22:09:25 +01:00 committed by GitHub
parent 6c19beb937
commit 409721414b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 132 additions and 83 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
Dockerfile
View file

@ -1,3 +1,4 @@
# vim: set ft=dockerfile:
ARG BUILD_ENV=full
ARG GOVERSION=1.19
@ -22,7 +23,8 @@ FROM alpine:latest as build-slim
RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata yq bash && \
mkdir -p /staging/etc/crowdsec && \
mkdir -p /staging/var/lib/crowdsec && \
mkdir -p /var/lib/crowdsec/data
mkdir -p /var/lib/crowdsec/data \
yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml
COPY --from=build /etc/crowdsec /staging/etc/crowdsec
COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
@ -30,17 +32,21 @@ COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
# NOTE: setting default values here will overwrite the ones set in config.yaml
# every time the container is started. We set the default in docker/config.yaml
# and document them in docker/README.md, but keep the variables empty here.
ENV CONFIG_FILE=/etc/crowdsec/config.yaml
ENV LOCAL_API_URL=http://0.0.0.0:8080/
ENV LOCAL_API_URL=
ENV CUSTOM_HOSTNAME=localhost
ENV PLUGIN_DIR=/usr/local/lib/crowdsec/plugins/
ENV PLUGIN_DIR=
ENV DISABLE_AGENT=false
ENV DISABLE_LOCAL_API=false
ENV DISABLE_ONLINE_API=false
ENV DSN=
ENV TYPE=
ENV TEST_MODE=false
ENV USE_WAL=false
ENV USE_WAL=
# register to app.crowdsec.net
@ -50,9 +56,9 @@ ENV ENROLL_TAGS=
# log verbosity
ENV LEVEL_TRACE=false
ENV LEVEL_DEBUG=false
ENV LEVEL_INFO=true
ENV LEVEL_TRACE=
ENV LEVEL_DEBUG=
ENV LEVEL_INFO=
# TLS setup ----------------------------------- #
@ -62,13 +68,13 @@ ENV AGENT_PASSWORD=
# TLS setup ----------------------------------- #
ENV USE_TLS=false
ENV CA_CERT_PATH=
ENV CERT_FILE=/etc/ssl/cert.pem
ENV KEY_FILE=/etc/ssl/key.pem
ENV CACERT_FILE=
ENV CERT_FILE=
ENV KEY_FILE=
# comma-separated list of allowed OU values for TLS bouncer certificates
ENV BOUNCERS_ALLOWED_OU=bouncer-ou
ENV BOUNCERS_ALLOWED_OU=
# comma-separated list of allowed OU values for TLS agent certificates
ENV AGENTS_ALLOWED_OU=agent-ou
ENV AGENTS_ALLOWED_OU=
# Install the following hub items --------------#
@ -84,7 +90,7 @@ ENV DISABLE_PARSERS=
ENV DISABLE_SCENARIOS=
ENV DISABLE_POSTOVERFLOWS=
ENV METRICS_PORT=6060
ENV METRICS_PORT=
ENTRYPOINT /bin/bash docker_start.sh

38
Dockerfile.debian
View file

@ -8,9 +8,12 @@ WORKDIR /go/src/crowdsec
COPY . .
ENV DEBIAN_FRONTEND=noninteractive
ENV DEBCONF_NOWARNINGS="yes"
# wizard.sh requires GNU coreutils
RUN apt-get update && \
apt-get install -y git gcc libc-dev make bash gettext binutils-gold coreutils tzdata && \
apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata && \
SYSTEM="docker" make release && \
cd crowdsec-v* && \
./wizard.sh --docker-mode && \
@ -18,7 +21,7 @@ RUN apt-get update && \
cscli hub update && \
cscli collections install crowdsecurity/linux && \
cscli parsers install crowdsecurity/whitelists && \
go install github.com/mikefarah/yq/v4@latest
go install github.com/mikefarah/yq/v4@v4.30.5
FROM debian:bullseye-slim as build-slim
@ -32,7 +35,8 @@ RUN apt-get update && \
tzdata && \
mkdir -p /staging/etc/crowdsec && \
mkdir -p /staging/var/lib/crowdsec && \
mkdir -p /var/lib/crowdsec/data
mkdir -p /var/lib/crowdsec/data \
yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml
COPY --from=build /go/bin/yq /usr/local/bin/yq
COPY --from=build /etc/crowdsec /staging/etc/crowdsec
@ -42,17 +46,21 @@ COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
RUN yq eval -i ".plugin_config.group = \"nogroup\"" /staging/etc/crowdsec/config.yaml
# NOTE: setting default values here will overwrite the ones set in config.yaml
# every time the container is started. We set the default in docker/config.yaml
# and document them in docker/README.md, but keep the variables empty here.
ENV CONFIG_FILE=/etc/crowdsec/config.yaml
ENV LOCAL_API_URL=http://0.0.0.0:8080/
ENV LOCAL_API_URL=
ENV CUSTOM_HOSTNAME=localhost
ENV PLUGIN_DIR=/usr/local/lib/crowdsec/plugins/
ENV PLUGIN_DIR=
ENV DISABLE_AGENT=false
ENV DISABLE_LOCAL_API=false
ENV DISABLE_ONLINE_API=false
ENV DSN=
ENV TYPE=
ENV TEST_MODE=false
ENV USE_WAL=false
ENV USE_WAL=
# register to app.crowdsec.net
@ -62,9 +70,9 @@ ENV ENROLL_TAGS=
# log verbosity
ENV LEVEL_TRACE=false
ENV LEVEL_DEBUG=false
ENV LEVEL_INFO=true
ENV LEVEL_TRACE=
ENV LEVEL_DEBUG=
ENV LEVEL_INFO=
# TLS setup ----------------------------------- #
@ -74,13 +82,13 @@ ENV AGENT_PASSWORD=
# TLS setup ----------------------------------- #
ENV USE_TLS=false
ENV CA_CERT_PATH=
ENV CERT_FILE=/etc/ssl/cert.pem
ENV KEY_FILE=/etc/ssl/key.pem
ENV CACERT_FILE=
ENV CERT_FILE=
ENV KEY_FILE=
# comma-separated list of allowed OU values for TLS bouncer certificates
ENV BOUNCERS_ALLOWED_OU=bouncer-ou
ENV BOUNCERS_ALLOWED_OU=
# comma-separated list of allowed OU values for TLS agent certificates
ENV AGENTS_ALLOWED_OU=agent-ou
ENV AGENTS_ALLOWED_OU=
# Install the following hub items --------------#
@ -96,7 +104,7 @@ ENV DISABLE_PARSERS=
ENV DISABLE_SCENARIOS=
ENV DISABLE_POSTOVERFLOWS=
ENV METRICS_PORT=6060
ENV METRICS_PORT=
ENTRYPOINT /bin/bash docker_start.sh

16
docker/config.yaml
View file

@ -24,14 +24,10 @@ db_config:
log_level: info
type: sqlite
db_path: /var/lib/crowdsec/data/crowdsec.db
#user:
#password:
#db_name:
#host:
#port:
flush:
max_items: 5000
max_age: 7d
use_wal: false
api:
client:
insecure_skip_verify: false
@ -45,9 +41,13 @@ api:
- ::1
online_client: # Central API credentials (to push signals and receive bad IPs)
#credentials_path: /etc/crowdsec/online_api_credentials.yaml
# tls:
# cert_file: /etc/crowdsec/ssl/cert.pem
# key_file: /etc/crowdsec/ssl/key.pem
tls:
cert_file: /etc/ssl/cert.pem
key_file: /etc/ssl/key.pem
agents_allowed_ou:
- agent-ou
bouncers_allowed_ou:
- bouncer-ou
prometheus:
enabled: true
level: full

129
docker/docker_start.sh
View file

@ -3,11 +3,15 @@
# shellcheck disable=SC2292 # allow [ test ] syntax
# shellcheck disable=SC2310 # allow "if function..." syntax with -e
#set -x
#export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
set -e
shopt -s inherit_errexit
#- HELPER FUNCTIONS ----------------#
# match true, TRUE, True, tRuE, etc.
istrue() {
case "$(echo "$1" | tr '[:upper:]' '[:lower:]')" in
true) return 0 ;;
@ -23,6 +27,7 @@ isfalse() {
fi
}
# csv2yaml <string>
# generate a yaml list from a comma-separated string of values
csv2yaml() {
[ -z "$1" ] && return
@ -34,6 +39,8 @@ cscli() {
command cscli -c "$CONFIG_FILE" "$@"
}
# conf_get <key> [file_path]
# retrieve a value from a file (by default $CONFIG_FILE)
conf_get() {
if [ $# -ge 2 ]; then
yq e "$1" "$2"
@ -42,12 +49,43 @@ conf_get() {
fi
}
# conf_set <yq_expression> [file_path]
# evaluate a yq command (by default on $CONFIG_FILE),
# create the file if it doesn't exist
conf_set() {
if [ $# -ge 2 ]; then
yq e "$1" -i "$2"
YAML_FILE="$2"
else
yq e "$1" -i "$CONFIG_FILE"
YAML_FILE="$CONFIG_FILE"
fi
YAML_CONTENT=$(cat "$YAML_FILE" 2>/dev/null || true)
echo "$YAML_CONTENT" | yq e "$1" | install -m 0600 /dev/stdin "$YAML_FILE"
}
# register_bouncer <bouncer_name> <bouncer_key>
register_bouncer() {
if ! cscli bouncers list -o json | sed '/^ *"name"/!d;s/^ *"name": "\(.*\)",/\1/' | grep -q "^${1}$"; then
if cscli bouncers add "$1" -k "$2" > /dev/null; then
echo "Registered bouncer for $1"
else
echo "Failed to register bouncer for $1"
fi
fi
}
# Call cscli to manage objects ignoring taint errors
# $1 can be collections, parsers, etc.
# $2 can be install, remove, upgrade
# $3 is a list of object names separated by space
cscli_if_clean() {
# loop over all objects
for obj in $3; do
if cscli "$1" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then
echo "Object $1/$obj is tainted, skipping"
else
cscli "$1" "$2" "$obj"
fi
done
}
#-----------------------------------#
@ -89,26 +127,30 @@ if isfalse "$DISABLE_AGENT"; then
if isfalse "$DISABLE_LOCAL_API"; then
echo "Regenerate local agent credentials"
cscli machines delete "$CUSTOM_HOSTNAME" 2>/dev/null || true
# shellcheck disable=SC2086
cscli machines add "$CUSTOM_HOSTNAME" --auto --url "$LOCAL_API_URL"
fi
lapi_credentials_path=$(conf_get '.api.client.credentials_path')
if istrue "$USE_TLS"; then
install -m 0600 /dev/null "$lapi_credentials_path"
conf_set '
.url = strenv(LOCAL_API_URL) |
.ca_cert_path = strenv(CACERT_FILE) |
.key_path = strenv(KEY_FILE) |
.cert_path = strenv(CERT_FILE)
# we only use the envvars that are actually defined
# in case of persistent configuration
conf_set '
with(select(strenv(LOCAL_API_URL)!=""); .url = strenv(LOCAL_API_URL)) |
with(select(strenv(AGENT_USERNAME)!=""); .login = strenv(AGENT_USERNAME)) |
with(select(strenv(AGENT_PASSWORD)!=""); .password = strenv(AGENT_PASSWORD))
' "$lapi_credentials_path"
elif [ "$AGENT_USERNAME" != "" ]; then
install -m 0600 /dev/null "$lapi_credentials_path"
if istrue "$USE_TLS"; then
conf_set '
.url = strenv(LOCAL_API_URL) |
.login = strenv(AGENT_USERNAME) |
.password = strenv(AGENT_PASSWORD)
with(select(strenv(CACERT_FILE)!=""); .ca_cert_path = strenv(CACERT_FILE)) |
with(select(strenv(KEY_FILE)!=""); .key_path = strenv(KEY_FILE)) |
with(select(strenv(CERT_FILE)!=""); .cert_path = strenv(CERT_FILE))
' "$lapi_credentials_path"
else
conf_set '
del(.ca_cert_path) |
del(.key_path) |
del(.cert_path)
' "$lapi_credentials_path"
fi
fi
@ -118,8 +160,8 @@ if isfalse "$DISABLE_LOCAL_API"; then
# pre-registration is not needed with TLS
if isfalse "$USE_TLS" && [ "$AGENT_USERNAME" != "" ] && [ "$AGENT_PASSWORD" != "" ] ; then
# shellcheck disable=SC2086
cscli machines add "$AGENT_USERNAME" --password "$AGENT_PASSWORD" --url "$LOCAL_API_URL"
# re-register because pw may have been changed
cscli machines add "$AGENT_USERNAME" --password "$AGENT_PASSWORD" --url "$LOCAL_API_URL" --force
echo "Agent registered to lapi"
fi
fi
@ -162,80 +204,73 @@ if istrue "$USE_TLS"; then
agents_allowed_yaml=$(csv2yaml "$AGENTS_ALLOWED_OU") \
bouncers_allowed_yaml=$(csv2yaml "$BOUNCERS_ALLOWED_OU") \
conf_set '
.api.server.tls.ca_cert_path = strenv(CACERT_FILE) |
.api.server.tls.cert_file = strenv(CERT_FILE) |
.api.server.tls.key_file = strenv(KEY_FILE) |
.api.server.tls.bouncers_allowed_ou = env(bouncers_allowed_yaml) |
.api.server.tls.agents_allowed_ou = env(agents_allowed_yaml) |
with(select(strenv(CACERT_FILE)!=""); .api.server.tls.ca_cert_path = strenv(CACERT_FILE)) |
with(select(strenv(CERT_FILE)!=""); .api.server.tls.cert_file = strenv(CERT_FILE)) |
with(select(strenv(KEY_FILE)!=""); .api.server.tls.key_file = strenv(KEY_FILE)) |
with(select(strenv(BOUNCERS_ALLOWED_OU)!=""); .api.server.tls.bouncers_allowed_ou = env(bouncers_allowed_yaml)) |
with(select(strenv(AGENTS_ALLOWED_OU)!=""); .api.server.tls.agents_allowed_ou = env(agents_allowed_yaml)) |
... comments=""
'
else
conf_set 'del(.api.server.tls)'
fi
conf_set ".config_paths.plugin_dir = strenv(PLUGIN_DIR)"
conf_set 'with(select(strenv(PLUGIN_DIR)!=""); .config_paths.plugin_dir = strenv(PLUGIN_DIR))'
## Install collections, parsers, scenarios & postoverflows
cscli hub update
cscli collections upgrade crowdsecurity/linux || true
cscli parsers upgrade crowdsecurity/whitelists || true
cscli parsers install crowdsecurity/docker-logs || true
cscli_if_clean collections upgrade crowdsecurity/linux
cscli_if_clean parsers upgrade crowdsecurity/whitelists
cscli_if_clean parsers install crowdsecurity/docker-logs
if [ "$COLLECTIONS" != "" ]; then
# shellcheck disable=SC2086
cscli collections install $COLLECTIONS
cscli_if_clean collections install $COLLECTIONS
fi
if [ "$PARSERS" != "" ]; then
# shellcheck disable=SC2086
cscli parsers install $PARSERS
cscli_if_clean parsers install $PARSERS
fi
if [ "$SCENARIOS" != "" ]; then
# shellcheck disable=SC2086
cscli scenarios install $SCENARIOS
cscli_if_clean scenarios install $SCENARIOS
fi
if [ "$POSTOVERFLOWS" != "" ]; then
# shellcheck disable=SC2086
cscli postoverflows install $POSTOVERFLOWS
cscli_if_clean postoverflows install $POSTOVERFLOWS
fi
## Remove collections, parsers, scenarios & postoverflows
if [ "$DISABLE_COLLECTIONS" != "" ]; then
# shellcheck disable=SC2086
cscli collections remove $DISABLE_COLLECTIONS
cscli_if_clean collections remove $DISABLE_COLLECTIONS
fi
if [ "$DISABLE_PARSERS" != "" ]; then
# shellcheck disable=SC2086
cscli parsers remove $DISABLE_PARSERS
cscli_if_clean parsers remove $DISABLE_PARSERS
fi
if [ "$DISABLE_SCENARIOS" != "" ]; then
# shellcheck disable=SC2086
cscli scenarios remove $DISABLE_SCENARIOS
cscli_if_clean scenarios remove $DISABLE_SCENARIOS
fi
if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
# shellcheck disable=SC2086
cscli postoverflows remove $DISABLE_POSTOVERFLOWS
cscli_if_clean postoverflows remove $DISABLE_POSTOVERFLOWS
fi
register_bouncer() {
if ! cscli bouncers list -o json | sed '/^ *"name"/!d;s/^ *"name": "\(.*\)",/\1/' | grep -q "^${NAME}$"; then
if cscli bouncers add "${NAME}" -k "${KEY}" > /dev/null; then
echo "Registered bouncer for ${NAME}"
else
echo "Failed to register bouncer for ${NAME}"
fi
fi
}
## Register bouncers via env
for BOUNCER in $(compgen -A variable | grep -i BOUNCER_KEY); do
KEY=$(printf '%s' "${!BOUNCER}")
NAME=$(printf '%s' "$BOUNCER" | cut -d_ -f3-)
if [[ -n $KEY ]] && [[ -n $NAME ]]; then
register_bouncer
register_bouncer "$NAME" "$KEY"
fi
done
@ -245,7 +280,7 @@ for BOUNCER in /run/secrets/@(bouncer_key|BOUNCER_KEY)* ; do
KEY=$(cat "${BOUNCER}")
NAME=$(echo "${BOUNCER}" | awk -F "/" '{printf $NF}' | cut -d_ -f2-)
if [[ -n $KEY ]] && [[ -n $NAME ]]; then
register_bouncer
register_bouncer "$NAME" "$KEY"
fi
done
shopt -u nullglob extglob
@ -287,7 +322,7 @@ if istrue "$LEVEL_INFO"; then
ARGS="$ARGS -info"
fi
conf_set '.prometheus.listen_port=env(METRICS_PORT)'
conf_set 'with(select(strenv(METRICS_PORT)!=""); .prometheus.listen_port=env(METRICS_PORT))'
# shellcheck disable=SC2086
exec crowdsec $ARGS