Create debian docker package including journalctl/systemd (#1233)

* Create debian docker package with journalctl

Co-authored-by: he2ss <hamza.essahely@gmail.com>

.github/workflows/release_publish_docker-image-debian.yml

@@ -0,0 +1,59 @@
+name: Publish Docker Debian image
+on:
+  release:
+    types:
+      - released
+      - prereleased
+jobs:
+  push_to_registry:
+    name: Push Docker debian image to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Check out the repo
+        uses: actions/checkout@v2
+      -
+        name: Prepare
+        id: prep
+        run: |
+          DOCKER_IMAGE=crowdsecurity/crowdsec
+          VERSION=bullseye
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            VERSION=${GITHUB_REF#refs/tags/}
+          elif [[ $GITHUB_REF == refs/heads/* ]]; then
+            VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -E 's#/+#-#g')
+          elif [[ $GITHUB_REF == refs/pull/* ]]; then
+            VERSION=pr-${{ github.event.number }}
+          fi
+          TAGS="${DOCKER_IMAGE}:${VERSION}-debian"
+          if [[ ${{ github.event.action }} == released ]]; then
+            TAGS=$TAGS,${DOCKER_IMAGE}:latest-debian
+          fi
+          echo ::set-output name=version::${VERSION}
+          echo ::set-output name=tags::${TAGS}
+          echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfile.debian
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.prep.outputs.tags }}
+          platforms: linux/amd64,linux/arm64
+          labels: |
+            org.opencontainers.image.source=${{ github.event.repository.html_url }}
+            org.opencontainers.image.created=${{ steps.prep.outputs.created }}
+            org.opencontainers.image.revision=${{ github.sha }}

.github/workflows/release_publish_docker-image.yml

@@ -9,7 +9,7 @@ jobs:
     name: Push Docker image to Docker Hub
     runs-on: ubuntu-latest
     steps:
-      - 
+      -
         name: Check out the repo
         uses: actions/checkout@v2
       -
@@ -32,10 +32,10 @@ jobs:
           echo ::set-output name=version::${VERSION}
           echo ::set-output name=tags::${TAGS}
           echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ')
-      - 
+      -
         name: Set up QEMU
         uses: docker/setup-qemu-action@v1
-      - 
+      -
         name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
       -
@@ -56,4 +56,4 @@ jobs:
           labels: |
             org.opencontainers.image.source=${{ github.event.repository.html_url }}
             org.opencontainers.image.created=${{ steps.prep.outputs.created }}
-            org.opencontainers.image.revision=${{ github.sha }}
+            org.opencontainers.image.revision=${{ github.sha }}

.github/workflows/update_docker_hub_doc.yml

@@ -1,6 +1,6 @@
 name: Update Docker Hub README
 
-on: 
+on:
   push:
     branches:
       - master
@@ -11,7 +11,7 @@ jobs:
   update-docker-hub-readme:
     runs-on: ubuntu-latest
     steps:
-      - 
+      -
         name: Check out the repo
         uses: actions/checkout@v2
       -
@@ -21,4 +21,4 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           repository: crowdsecurity/crowdsec
-          readme: "./docker/README.md"
+          readme: "./docker/README.md"

Dockerfile

@@ -13,7 +13,7 @@ RUN SYSTEM="docker" make release
 RUN cd crowdsec-v* && ./wizard.sh --docker-mode && cd -
 RUN cscli hub update && cscli collections install crowdsecurity/linux && cscli parsers install crowdsecurity/whitelists
 FROM alpine:latest
-RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata yq && \
+RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata yq bash && \
   mkdir -p /staging/etc/crowdsec && \
   mkdir -p /staging/var/lib/crowdsec
 COPY --from=build /etc/crowdsec /staging/etc/crowdsec
@@ -29,4 +29,4 @@ COPY --from=build /go/src/crowdsec/plugins/notifications/slack/slack.yaml /stagi
 COPY --from=build /go/src/crowdsec/plugins/notifications/splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
 COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins
 
-ENTRYPOINT /bin/sh docker_start.sh
+ENTRYPOINT /bin/bash docker_start.sh

Dockerfile.debian

@@ -0,0 +1,45 @@
+ARG GOVERSION=1.17
+
+FROM golang:${GOVERSION}-bullseye AS build
+
+WORKDIR /go/src/crowdsec
+
+# wizard.sh requires GNU coreutils
+RUN apt-get update && apt-get install -y git jq gcc libc-dev make bash gettext binutils-gold coreutils tzdata python3 python3-pip
+
+COPY . .
+
+RUN SYSTEM="docker" make release
+RUN cd crowdsec-v* && ./wizard.sh --docker-mode && cd -
+RUN cscli hub update && cscli collections install crowdsecurity/linux && cscli parsers install crowdsecurity/whitelists
+RUN GO111MODULE=on go get github.com/mikefarah/yq/v4
+
+FROM debian:bullseye-slim
+
+RUN apt-get update
+RUN apt-get install -y -q --install-recommends --no-install-suggests \
+    procps \
+    systemd \
+    iproute2 \
+    ca-certificates \
+    bash \
+    tzdata && \
+    mkdir -p /staging/etc/crowdsec && \
+    mkdir -p /staging/var/lib/crowdsec
+
+COPY --from=build /go/bin/yq /usr/local/bin/yq
+COPY --from=build /etc/crowdsec /staging/etc/crowdsec
+COPY --from=build /var/lib/crowdsec /staging/var/lib/crowdsec
+COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
+COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
+COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
+COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
+RUN yq eval -i ".plugin_config.group = \"nogroup\"" /staging/etc/crowdsec/config.yaml
+#Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
+#The files are here for reference, as users will need to mount a new version to be actually able to use notifications
+COPY --from=build /go/src/crowdsec/plugins/notifications/http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
+COPY --from=build /go/src/crowdsec/plugins/notifications/slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
+COPY --from=build /go/src/crowdsec/plugins/notifications/splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
+COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins
+
+ENTRYPOINT /bin/bash docker_start.sh

docker/README.md

@@ -4,15 +4,25 @@
 * Crowdsec concepts: https://docs.crowdsec.net/docs/concepts
 * Where to file issues: https://github.com/crowdsecurity/crowdsec
 
-
 # What is Crowdsec
 
 Crowdsec - An open-source, lightweight agent to detect and respond to bad behaviours. It also automatically benefits from our global community-wide IP reputation database.
 
 # How to use this image
 
+## Docker images available
+crowdsec will use Alpine as default container. A debian container is also available with systemd for journalctl support. Simply add `-debian` to your tag to use this. Please be aware that debian containers are not available on all version, since the feature was implemented after the release of version 1.3.0
+
 ## Required configuration
 
+### Journalctl
+To use journalctl as log stream, eventually from the `DSN` environment variable, it's important that you mount the journal log from the host to the container it self.
+This can be done by adding the following volume mount to your docker command:
+
+```
+-v /var/log/journal:/run/log/journal
+```
+
 ### Logs ingestion and processing
 Collections are a good place to start: https://docs.crowdsec.net/docs/collections/intro
 
@@ -127,6 +137,11 @@ Using binds rather than named volumes ([more explanation here](https://docs.dock
 * `CERT_FILE`               - TLS Certificate file (default: `/etc/ssl/cert.pem`) : `-e CERT_FILE="<file_path>"`
 * `KEY_FILE`                - TLS Key file (default: `/etc/ssl/key.pem`) : `-e KEY_FILE="<file_path>"`
 * `CUSTOM_HOSTNAME`         - Custom hostname for local api (default: `localhost`) : `-e CUSTOM_HOSTNAME="<hostname>"`
+* `DISABLE_COLLECTIONS`       - Collections to remove from the [hub](https://hub.crowdsec.net/browse/#collections), separated by space : `-e DISABLE_COLLECTIONS="crowdsecurity/linux crowdsecurity/nginx"`
+* `DISABLE_PARSERS`         - Parsers to remove from the [hub](https://hub.crowdsec.net/browse/#configurations), separated by space : `-e DISABLE_PARSERS="crowdsecurity/apache2-logs crowdsecurity/nginx-logs"`
+* `DISABLE_SCENARIOS`       - Scenarios to remove from the [hub](https://hub.crowdsec.net/browse/#configurations), separated by space : `-e DISABLE_SCENARIOS="crowdsecurity/http-bad-user-agent crowdsecurity/http-xss-probing"`
+* `DISABLE_POSTOVERFLOWS`   - Postoverflows to remove from the [hub](https://hub.crowdsec.net/browse/#configurations), separated by space : `-e DISABLE_POSTOVERFLOWS="crowdsecurity/cdn-whitelist crowdsecurity/seo-bots-whitelist"`
+* `PLUGIN_DIR`              - Directory for plugins (default: `/usr/local/lib/crowdsec/plugins/`) : `-e PLUGIN_DIR="<path>"`
 
 ## Volumes
 
@@ -137,7 +152,7 @@ Using binds rather than named volumes ([more explanation here](https://docs.dock
 ## File Locations
 
 * `/usr/local/bin/crowdsec` - Crowdsec binary
-  
+
 * `/usr/local/bin/cscli` - Crowdsec CLI binary to interact with crowdsec
 
 # Find Us

docker/docker_start.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # Set the crowdsec config file
 CS_CONFIG_FILE="/etc/crowdsec/config.yaml"
@@ -10,6 +10,9 @@ fi
 CERT_FILE="${CERT_FILE:-/etc/ssl/cert.pem}"
 KEY_FILE="${KEY_FILE:-/etc/ssl/key.pem}"
 
+# Plugins directory default
+PLUGIN_DIR="${PLUGIN_DIR:-/usr/local/lib/crowdsec/plugins/}"
+
 #Check & prestage databases
 if [ ! -e "/var/lib/data/GeoLite2-ASN.mmdb" ] && [ ! -e "/var/lib/data/GeoLite2-City.mmdb" ]; then
     mkdir -p /var/lib/crowdsec/data
@@ -41,7 +44,7 @@ if [ "$DISABLE_AGENT" == "" ] ; then
 fi
 
 # Check if lapi needs to automatically register an agent
-echo Check if lapi need to register automatically an agent
+echo "Check if lapi need to register automatically an agent"
 if [ "$DISABLE_LOCAL_API" == "" ] && [ "$AGENT_USERNAME" != "" ] && [ "$AGENT_PASSWORD" != "" ] ; then
     if [ "$LOCAL_API_URL" != "" ] ; then
         cscli -c "$CS_CONFIG_FILE" machines add $AGENT_USERNAME --password $AGENT_PASSWORD --url $LOCAL_API_URL
@@ -78,7 +81,11 @@ if [ "$USE_TLS" != "" ]; then
    yq -i eval '... comments=""' "$CS_CONFIG_FILE"
 fi
 
-## Install collections, parsers & scenarios
+if [ "$PLUGIN_DIR" != "/usr/local/lib/crowdsec/plugins/" ]; then
+   yq -i eval ".config_paths.plugin_dir = \"$PLUGIN_DIR\"" "$CS_CONFIG_FILE"
+fi
+
+## Install collections, parsers, scenarios & postoverflows
 cscli -c "$CS_CONFIG_FILE" hub update
 cscli -c "$CS_CONFIG_FILE" collections upgrade crowdsecurity/linux || true
 cscli -c "$CS_CONFIG_FILE" parsers upgrade crowdsecurity/whitelists || true
@@ -96,6 +103,20 @@ if [ "$POSTOVERFLOWS" != "" ]; then
     cscli -c "$CS_CONFIG_FILE" postoverflows install $POSTOVERFLOWS
 fi
 
+## Remove collections, parsers, scenarios & postoverflows
+if [ "$DISABLE_COLLECTIONS" != "" ]; then
+    cscli -c "$CS_CONFIG_FILE" collections remove $DISABLE_COLLECTIONS
+fi
+if [ "$DISABLE_PARSERS" != "" ]; then
+    cscli -c "$CS_CONFIG_FILE" parsers remove $DISABLE_PARSERS
+fi
+if [ "$DISABLE_SCENARIOS" != "" ]; then
+    cscli -c "$CS_CONFIG_FILE" scenarios remove $DISABLE_SCENARIOS
+fi
+if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
+    cscli -c "$CS_CONFIG_FILE" postoverflows remove $DISABLE_POSTOVERFLOWS
+fi
+
 ARGS=""
 if [ "$CONFIG_FILE" != "" ]; then
     ARGS="-c $CONFIG_FILE"