diff --git a/Dockerfile b/Dockerfile index 7f327b870..b1a35e399 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,4 @@ +# vim: set ft=dockerfile: ARG BUILD_ENV=full ARG GOVERSION=1.19 @@ -22,7 +23,8 @@ FROM alpine:latest as build-slim RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata yq bash && \ mkdir -p /staging/etc/crowdsec && \ mkdir -p /staging/var/lib/crowdsec && \ - mkdir -p /var/lib/crowdsec/data + mkdir -p /var/lib/crowdsec/data \ + yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml COPY --from=build /etc/crowdsec /staging/etc/crowdsec COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec @@ -30,17 +32,21 @@ COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli COPY --from=build /go/src/crowdsec/docker/docker_start.sh / COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml +# NOTE: setting default values here will overwrite the ones set in config.yaml +# every time the container is started. We set the default in docker/config.yaml +# and document them in docker/README.md, but keep the variables empty here. + ENV CONFIG_FILE=/etc/crowdsec/config.yaml -ENV LOCAL_API_URL=http://0.0.0.0:8080/ +ENV LOCAL_API_URL= ENV CUSTOM_HOSTNAME=localhost -ENV PLUGIN_DIR=/usr/local/lib/crowdsec/plugins/ +ENV PLUGIN_DIR= ENV DISABLE_AGENT=false ENV DISABLE_LOCAL_API=false ENV DISABLE_ONLINE_API=false ENV DSN= ENV TYPE= ENV TEST_MODE=false -ENV USE_WAL=false +ENV USE_WAL= # register to app.crowdsec.net @@ -50,9 +56,9 @@ ENV ENROLL_TAGS= # log verbosity -ENV LEVEL_TRACE=false -ENV LEVEL_DEBUG=false -ENV LEVEL_INFO=true +ENV LEVEL_TRACE= +ENV LEVEL_DEBUG= +ENV LEVEL_INFO= # TLS setup ----------------------------------- # @@ -62,13 +68,13 @@ ENV AGENT_PASSWORD= # TLS setup ----------------------------------- # ENV USE_TLS=false -ENV CA_CERT_PATH= -ENV CERT_FILE=/etc/ssl/cert.pem -ENV KEY_FILE=/etc/ssl/key.pem +ENV CACERT_FILE= +ENV CERT_FILE= +ENV KEY_FILE= # comma-separated list of allowed OU values for TLS bouncer certificates -ENV BOUNCERS_ALLOWED_OU=bouncer-ou +ENV BOUNCERS_ALLOWED_OU= # comma-separated list of allowed OU values for TLS agent certificates -ENV AGENTS_ALLOWED_OU=agent-ou +ENV AGENTS_ALLOWED_OU= # Install the following hub items --------------# @@ -84,7 +90,7 @@ ENV DISABLE_PARSERS= ENV DISABLE_SCENARIOS= ENV DISABLE_POSTOVERFLOWS= -ENV METRICS_PORT=6060 +ENV METRICS_PORT= ENTRYPOINT /bin/bash docker_start.sh diff --git a/Dockerfile.debian b/Dockerfile.debian index 4ea120e3a..faebeb860 100644 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -8,9 +8,12 @@ WORKDIR /go/src/crowdsec COPY . . +ENV DEBIAN_FRONTEND=noninteractive +ENV DEBCONF_NOWARNINGS="yes" + # wizard.sh requires GNU coreutils RUN apt-get update && \ - apt-get install -y git gcc libc-dev make bash gettext binutils-gold coreutils tzdata && \ + apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata && \ SYSTEM="docker" make release && \ cd crowdsec-v* && \ ./wizard.sh --docker-mode && \ @@ -18,7 +21,7 @@ RUN apt-get update && \ cscli hub update && \ cscli collections install crowdsecurity/linux && \ cscli parsers install crowdsecurity/whitelists && \ - go install github.com/mikefarah/yq/v4@latest + go install github.com/mikefarah/yq/v4@v4.30.5 FROM debian:bullseye-slim as build-slim @@ -32,7 +35,8 @@ RUN apt-get update && \ tzdata && \ mkdir -p /staging/etc/crowdsec && \ mkdir -p /staging/var/lib/crowdsec && \ - mkdir -p /var/lib/crowdsec/data + mkdir -p /var/lib/crowdsec/data \ + yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml COPY --from=build /go/bin/yq /usr/local/bin/yq COPY --from=build /etc/crowdsec /staging/etc/crowdsec @@ -42,17 +46,21 @@ COPY --from=build /go/src/crowdsec/docker/docker_start.sh / COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml RUN yq eval -i ".plugin_config.group = \"nogroup\"" /staging/etc/crowdsec/config.yaml +# NOTE: setting default values here will overwrite the ones set in config.yaml +# every time the container is started. We set the default in docker/config.yaml +# and document them in docker/README.md, but keep the variables empty here. + ENV CONFIG_FILE=/etc/crowdsec/config.yaml -ENV LOCAL_API_URL=http://0.0.0.0:8080/ +ENV LOCAL_API_URL= ENV CUSTOM_HOSTNAME=localhost -ENV PLUGIN_DIR=/usr/local/lib/crowdsec/plugins/ +ENV PLUGIN_DIR= ENV DISABLE_AGENT=false ENV DISABLE_LOCAL_API=false ENV DISABLE_ONLINE_API=false ENV DSN= ENV TYPE= ENV TEST_MODE=false -ENV USE_WAL=false +ENV USE_WAL= # register to app.crowdsec.net @@ -62,9 +70,9 @@ ENV ENROLL_TAGS= # log verbosity -ENV LEVEL_TRACE=false -ENV LEVEL_DEBUG=false -ENV LEVEL_INFO=true +ENV LEVEL_TRACE= +ENV LEVEL_DEBUG= +ENV LEVEL_INFO= # TLS setup ----------------------------------- # @@ -74,13 +82,13 @@ ENV AGENT_PASSWORD= # TLS setup ----------------------------------- # ENV USE_TLS=false -ENV CA_CERT_PATH= -ENV CERT_FILE=/etc/ssl/cert.pem -ENV KEY_FILE=/etc/ssl/key.pem +ENV CACERT_FILE= +ENV CERT_FILE= +ENV KEY_FILE= # comma-separated list of allowed OU values for TLS bouncer certificates -ENV BOUNCERS_ALLOWED_OU=bouncer-ou +ENV BOUNCERS_ALLOWED_OU= # comma-separated list of allowed OU values for TLS agent certificates -ENV AGENTS_ALLOWED_OU=agent-ou +ENV AGENTS_ALLOWED_OU= # Install the following hub items --------------# @@ -96,7 +104,7 @@ ENV DISABLE_PARSERS= ENV DISABLE_SCENARIOS= ENV DISABLE_POSTOVERFLOWS= -ENV METRICS_PORT=6060 +ENV METRICS_PORT= ENTRYPOINT /bin/bash docker_start.sh diff --git a/docker/config.yaml b/docker/config.yaml index 1589a31b3..33d53e7cd 100644 --- a/docker/config.yaml +++ b/docker/config.yaml @@ -25,14 +25,10 @@ db_config: log_level: info type: sqlite db_path: /var/lib/crowdsec/data/crowdsec.db - #user: - #password: - #db_name: - #host: - #port: flush: max_items: 5000 max_age: 7d + use_wal: false api: client: insecure_skip_verify: false @@ -46,9 +42,13 @@ api: - ::1 online_client: # Central API credentials (to push signals and receive bad IPs) #credentials_path: /etc/crowdsec/online_api_credentials.yaml -# tls: -# cert_file: /etc/crowdsec/ssl/cert.pem -# key_file: /etc/crowdsec/ssl/key.pem + tls: + cert_file: /etc/ssl/cert.pem + key_file: /etc/ssl/key.pem + agents_allowed_ou: + - agent-ou + bouncers_allowed_ou: + - bouncer-ou prometheus: enabled: true level: full diff --git a/docker/docker_start.sh b/docker/docker_start.sh index a43c36cfe..ae7a28beb 100755 --- a/docker/docker_start.sh +++ b/docker/docker_start.sh @@ -3,11 +3,15 @@ # shellcheck disable=SC2292 # allow [ test ] syntax # shellcheck disable=SC2310 # allow "if function..." syntax with -e +#set -x +#export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' + set -e shopt -s inherit_errexit #- HELPER FUNCTIONS ----------------# +# match true, TRUE, True, tRuE, etc. istrue() { case "$(echo "$1" | tr '[:upper:]' '[:lower:]')" in true) return 0 ;; @@ -23,6 +27,7 @@ isfalse() { fi } +# csv2yaml # generate a yaml list from a comma-separated string of values csv2yaml() { [ -z "$1" ] && return @@ -34,6 +39,8 @@ cscli() { command cscli -c "$CONFIG_FILE" "$@" } +# conf_get [file_path] +# retrieve a value from a file (by default $CONFIG_FILE) conf_get() { if [ $# -ge 2 ]; then yq e "$1" "$2" @@ -42,12 +49,43 @@ conf_get() { fi } +# conf_set [file_path] +# evaluate a yq command (by default on $CONFIG_FILE), +# create the file if it doesn't exist conf_set() { if [ $# -ge 2 ]; then - yq e "$1" -i "$2" + YAML_FILE="$2" else - yq e "$1" -i "$CONFIG_FILE" + YAML_FILE="$CONFIG_FILE" fi + YAML_CONTENT=$(cat "$YAML_FILE" 2>/dev/null || true) + echo "$YAML_CONTENT" | yq e "$1" | install -m 0600 /dev/stdin "$YAML_FILE" +} + +# register_bouncer +register_bouncer() { + if ! cscli bouncers list -o json | sed '/^ *"name"/!d;s/^ *"name": "\(.*\)",/\1/' | grep -q "^${1}$"; then + if cscli bouncers add "$1" -k "$2" > /dev/null; then + echo "Registered bouncer for $1" + else + echo "Failed to register bouncer for $1" + fi + fi +} + +# Call cscli to manage objects ignoring taint errors +# $1 can be collections, parsers, etc. +# $2 can be install, remove, upgrade +# $3 is a list of object names separated by space +cscli_if_clean() { + # loop over all objects + for obj in $3; do + if cscli "$1" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then + echo "Object $1/$obj is tainted, skipping" + else + cscli "$1" "$2" "$obj" + fi + done } #-----------------------------------# @@ -89,26 +127,30 @@ if isfalse "$DISABLE_AGENT"; then if isfalse "$DISABLE_LOCAL_API"; then echo "Regenerate local agent credentials" cscli machines delete "$CUSTOM_HOSTNAME" 2>/dev/null || true - # shellcheck disable=SC2086 cscli machines add "$CUSTOM_HOSTNAME" --auto --url "$LOCAL_API_URL" fi lapi_credentials_path=$(conf_get '.api.client.credentials_path') - if istrue "$USE_TLS"; then - install -m 0600 /dev/null "$lapi_credentials_path" - conf_set ' - .url = strenv(LOCAL_API_URL) | - .ca_cert_path = strenv(CACERT_FILE) | - .key_path = strenv(KEY_FILE) | - .cert_path = strenv(CERT_FILE) + # we only use the envvars that are actually defined + # in case of persistent configuration + conf_set ' + with(select(strenv(LOCAL_API_URL)!=""); .url = strenv(LOCAL_API_URL)) | + with(select(strenv(AGENT_USERNAME)!=""); .login = strenv(AGENT_USERNAME)) | + with(select(strenv(AGENT_PASSWORD)!=""); .password = strenv(AGENT_PASSWORD)) ' "$lapi_credentials_path" - elif [ "$AGENT_USERNAME" != "" ]; then - install -m 0600 /dev/null "$lapi_credentials_path" + + if istrue "$USE_TLS"; then conf_set ' - .url = strenv(LOCAL_API_URL) | - .login = strenv(AGENT_USERNAME) | - .password = strenv(AGENT_PASSWORD) + with(select(strenv(CACERT_FILE)!=""); .ca_cert_path = strenv(CACERT_FILE)) | + with(select(strenv(KEY_FILE)!=""); .key_path = strenv(KEY_FILE)) | + with(select(strenv(CERT_FILE)!=""); .cert_path = strenv(CERT_FILE)) + ' "$lapi_credentials_path" + else + conf_set ' + del(.ca_cert_path) | + del(.key_path) | + del(.cert_path) ' "$lapi_credentials_path" fi fi @@ -118,8 +160,8 @@ if isfalse "$DISABLE_LOCAL_API"; then # pre-registration is not needed with TLS if isfalse "$USE_TLS" && [ "$AGENT_USERNAME" != "" ] && [ "$AGENT_PASSWORD" != "" ] ; then - # shellcheck disable=SC2086 - cscli machines add "$AGENT_USERNAME" --password "$AGENT_PASSWORD" --url "$LOCAL_API_URL" + # re-register because pw may have been changed + cscli machines add "$AGENT_USERNAME" --password "$AGENT_PASSWORD" --url "$LOCAL_API_URL" --force echo "Agent registered to lapi" fi fi @@ -162,80 +204,73 @@ if istrue "$USE_TLS"; then agents_allowed_yaml=$(csv2yaml "$AGENTS_ALLOWED_OU") \ bouncers_allowed_yaml=$(csv2yaml "$BOUNCERS_ALLOWED_OU") \ conf_set ' - .api.server.tls.ca_cert_path = strenv(CACERT_FILE) | - .api.server.tls.cert_file = strenv(CERT_FILE) | - .api.server.tls.key_file = strenv(KEY_FILE) | - .api.server.tls.bouncers_allowed_ou = env(bouncers_allowed_yaml) | - .api.server.tls.agents_allowed_ou = env(agents_allowed_yaml) | + with(select(strenv(CACERT_FILE)!=""); .api.server.tls.ca_cert_path = strenv(CACERT_FILE)) | + with(select(strenv(CERT_FILE)!=""); .api.server.tls.cert_file = strenv(CERT_FILE)) | + with(select(strenv(KEY_FILE)!=""); .api.server.tls.key_file = strenv(KEY_FILE)) | + with(select(strenv(BOUNCERS_ALLOWED_OU)!=""); .api.server.tls.bouncers_allowed_ou = env(bouncers_allowed_yaml)) | + with(select(strenv(AGENTS_ALLOWED_OU)!=""); .api.server.tls.agents_allowed_ou = env(agents_allowed_yaml)) | ... comments="" ' +else + conf_set 'del(.api.server.tls)' fi -conf_set ".config_paths.plugin_dir = strenv(PLUGIN_DIR)" +conf_set 'with(select(strenv(PLUGIN_DIR)!=""); .config_paths.plugin_dir = strenv(PLUGIN_DIR))' ## Install collections, parsers, scenarios & postoverflows cscli hub update -cscli collections upgrade crowdsecurity/linux || true -cscli parsers upgrade crowdsecurity/whitelists || true -cscli parsers install crowdsecurity/docker-logs || true + +cscli_if_clean collections upgrade crowdsecurity/linux +cscli_if_clean parsers upgrade crowdsecurity/whitelists +cscli_if_clean parsers install crowdsecurity/docker-logs if [ "$COLLECTIONS" != "" ]; then # shellcheck disable=SC2086 - cscli collections install $COLLECTIONS + cscli_if_clean collections install $COLLECTIONS fi if [ "$PARSERS" != "" ]; then # shellcheck disable=SC2086 - cscli parsers install $PARSERS + cscli_if_clean parsers install $PARSERS fi if [ "$SCENARIOS" != "" ]; then # shellcheck disable=SC2086 - cscli scenarios install $SCENARIOS + cscli_if_clean scenarios install $SCENARIOS fi if [ "$POSTOVERFLOWS" != "" ]; then # shellcheck disable=SC2086 - cscli postoverflows install $POSTOVERFLOWS + cscli_if_clean postoverflows install $POSTOVERFLOWS fi ## Remove collections, parsers, scenarios & postoverflows if [ "$DISABLE_COLLECTIONS" != "" ]; then # shellcheck disable=SC2086 - cscli collections remove $DISABLE_COLLECTIONS + cscli_if_clean collections remove $DISABLE_COLLECTIONS fi if [ "$DISABLE_PARSERS" != "" ]; then # shellcheck disable=SC2086 - cscli parsers remove $DISABLE_PARSERS + cscli_if_clean parsers remove $DISABLE_PARSERS fi if [ "$DISABLE_SCENARIOS" != "" ]; then # shellcheck disable=SC2086 - cscli scenarios remove $DISABLE_SCENARIOS + cscli_if_clean scenarios remove $DISABLE_SCENARIOS fi if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then # shellcheck disable=SC2086 - cscli postoverflows remove $DISABLE_POSTOVERFLOWS + cscli_if_clean postoverflows remove $DISABLE_POSTOVERFLOWS fi -register_bouncer() { - if ! cscli bouncers list -o json | sed '/^ *"name"/!d;s/^ *"name": "\(.*\)",/\1/' | grep -q "^${NAME}$"; then - if cscli bouncers add "${NAME}" -k "${KEY}" > /dev/null; then - echo "Registered bouncer for ${NAME}" - else - echo "Failed to register bouncer for ${NAME}" - fi - fi -} - ## Register bouncers via env for BOUNCER in $(compgen -A variable | grep -i BOUNCER_KEY); do KEY=$(printf '%s' "${!BOUNCER}") NAME=$(printf '%s' "$BOUNCER" | cut -d_ -f3-) if [[ -n $KEY ]] && [[ -n $NAME ]]; then - register_bouncer + register_bouncer "$NAME" "$KEY" fi done @@ -245,7 +280,7 @@ for BOUNCER in /run/secrets/@(bouncer_key|BOUNCER_KEY)* ; do KEY=$(cat "${BOUNCER}") NAME=$(echo "${BOUNCER}" | awk -F "/" '{printf $NF}' | cut -d_ -f2-) if [[ -n $KEY ]] && [[ -n $NAME ]]; then - register_bouncer + register_bouncer "$NAME" "$KEY" fi done shopt -u nullglob extglob @@ -287,7 +322,7 @@ if istrue "$LEVEL_INFO"; then ARGS="$ARGS -info" fi -conf_set '.prometheus.listen_port=env(METRICS_PORT)' +conf_set 'with(select(strenv(METRICS_PORT)!=""); .prometheus.listen_port=env(METRICS_PORT))' # shellcheck disable=SC2086 exec crowdsec $ARGS