From d18620858ef2d1c9ab2000105e529ea3e94e9b6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kr=C3=BCger?= Date: Tue, 15 Feb 2022 17:10:15 +0100 Subject: [PATCH] Create debian docker package including journalctl/systemd (#1233) * Create debian docker package with journalctl Co-authored-by: he2ss --- .../release_publish_docker-image-debian.yml | 59 +++++++++++++++++++ .../release_publish_docker-image.yml | 8 +-- .github/workflows/update_docker_hub_doc.yml | 6 +- Dockerfile | 4 +- Dockerfile.debian | 45 ++++++++++++++ docker/README.md | 19 +++++- docker/docker_start.sh | 27 ++++++++- 7 files changed, 154 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/release_publish_docker-image-debian.yml create mode 100644 Dockerfile.debian diff --git a/.github/workflows/release_publish_docker-image-debian.yml b/.github/workflows/release_publish_docker-image-debian.yml new file mode 100644 index 000000000..985890b8f --- /dev/null +++ b/.github/workflows/release_publish_docker-image-debian.yml @@ -0,0 +1,59 @@ +name: Publish Docker Debian image +on: + release: + types: + - released + - prereleased +jobs: + push_to_registry: + name: Push Docker debian image to Docker Hub + runs-on: ubuntu-latest + steps: + - + name: Check out the repo + uses: actions/checkout@v2 + - + name: Prepare + id: prep + run: | + DOCKER_IMAGE=crowdsecurity/crowdsec + VERSION=bullseye + if [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/} + elif [[ $GITHUB_REF == refs/heads/* ]]; then + VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -E 's#/+#-#g') + elif [[ $GITHUB_REF == refs/pull/* ]]; then + VERSION=pr-${{ github.event.number }} + fi + TAGS="${DOCKER_IMAGE}:${VERSION}-debian" + if [[ ${{ github.event.action }} == released ]]; then + TAGS=$TAGS,${DOCKER_IMAGE}:latest-debian + fi + echo ::set-output name=version::${VERSION} + echo ::set-output name=tags::${TAGS} + echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - + name: Build and push + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile.debian + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.prep.outputs.tags }} + platforms: linux/amd64,linux/arm64 + labels: | + org.opencontainers.image.source=${{ github.event.repository.html_url }} + org.opencontainers.image.created=${{ steps.prep.outputs.created }} + org.opencontainers.image.revision=${{ github.sha }} diff --git a/.github/workflows/release_publish_docker-image.yml b/.github/workflows/release_publish_docker-image.yml index f2bdab8d1..7ff968efc 100644 --- a/.github/workflows/release_publish_docker-image.yml +++ b/.github/workflows/release_publish_docker-image.yml @@ -9,7 +9,7 @@ jobs: name: Push Docker image to Docker Hub runs-on: ubuntu-latest steps: - - + - name: Check out the repo uses: actions/checkout@v2 - @@ -32,10 +32,10 @@ jobs: echo ::set-output name=version::${VERSION} echo ::set-output name=tags::${TAGS} echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') - - + - name: Set up QEMU uses: docker/setup-qemu-action@v1 - - + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - @@ -56,4 +56,4 @@ jobs: labels: | org.opencontainers.image.source=${{ github.event.repository.html_url }} org.opencontainers.image.created=${{ steps.prep.outputs.created }} - org.opencontainers.image.revision=${{ github.sha }} \ No newline at end of file + org.opencontainers.image.revision=${{ github.sha }} diff --git a/.github/workflows/update_docker_hub_doc.yml b/.github/workflows/update_docker_hub_doc.yml index 8e5d1a31f..9e6b65073 100644 --- a/.github/workflows/update_docker_hub_doc.yml +++ b/.github/workflows/update_docker_hub_doc.yml @@ -1,6 +1,6 @@ name: Update Docker Hub README -on: +on: push: branches: - master @@ -11,7 +11,7 @@ jobs: update-docker-hub-readme: runs-on: ubuntu-latest steps: - - + - name: Check out the repo uses: actions/checkout@v2 - @@ -21,4 +21,4 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} repository: crowdsecurity/crowdsec - readme: "./docker/README.md" \ No newline at end of file + readme: "./docker/README.md" diff --git a/Dockerfile b/Dockerfile index a2ca83afb..8ff3f4b55 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ RUN SYSTEM="docker" make release RUN cd crowdsec-v* && ./wizard.sh --docker-mode && cd - RUN cscli hub update && cscli collections install crowdsecurity/linux && cscli parsers install crowdsecurity/whitelists FROM alpine:latest -RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata yq && \ +RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata yq bash && \ mkdir -p /staging/etc/crowdsec && \ mkdir -p /staging/var/lib/crowdsec COPY --from=build /etc/crowdsec /staging/etc/crowdsec @@ -29,4 +29,4 @@ COPY --from=build /go/src/crowdsec/plugins/notifications/slack/slack.yaml /stagi COPY --from=build /go/src/crowdsec/plugins/notifications/splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins -ENTRYPOINT /bin/sh docker_start.sh +ENTRYPOINT /bin/bash docker_start.sh diff --git a/Dockerfile.debian b/Dockerfile.debian new file mode 100644 index 000000000..bd797947a --- /dev/null +++ b/Dockerfile.debian @@ -0,0 +1,45 @@ +ARG GOVERSION=1.17 + +FROM golang:${GOVERSION}-bullseye AS build + +WORKDIR /go/src/crowdsec + +# wizard.sh requires GNU coreutils +RUN apt-get update && apt-get install -y git jq gcc libc-dev make bash gettext binutils-gold coreutils tzdata python3 python3-pip + +COPY . . + +RUN SYSTEM="docker" make release +RUN cd crowdsec-v* && ./wizard.sh --docker-mode && cd - +RUN cscli hub update && cscli collections install crowdsecurity/linux && cscli parsers install crowdsecurity/whitelists +RUN GO111MODULE=on go get github.com/mikefarah/yq/v4 + +FROM debian:bullseye-slim + +RUN apt-get update +RUN apt-get install -y -q --install-recommends --no-install-suggests \ + procps \ + systemd \ + iproute2 \ + ca-certificates \ + bash \ + tzdata && \ + mkdir -p /staging/etc/crowdsec && \ + mkdir -p /staging/var/lib/crowdsec + +COPY --from=build /go/bin/yq /usr/local/bin/yq +COPY --from=build /etc/crowdsec /staging/etc/crowdsec +COPY --from=build /var/lib/crowdsec /staging/var/lib/crowdsec +COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec +COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli +COPY --from=build /go/src/crowdsec/docker/docker_start.sh / +COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml +RUN yq eval -i ".plugin_config.group = \"nogroup\"" /staging/etc/crowdsec/config.yaml +#Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp +#The files are here for reference, as users will need to mount a new version to be actually able to use notifications +COPY --from=build /go/src/crowdsec/plugins/notifications/http/http.yaml /staging/etc/crowdsec/notifications/http.yaml +COPY --from=build /go/src/crowdsec/plugins/notifications/slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml +COPY --from=build /go/src/crowdsec/plugins/notifications/splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml +COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins + +ENTRYPOINT /bin/bash docker_start.sh diff --git a/docker/README.md b/docker/README.md index 478e3f41f..b3b5d3d6e 100644 --- a/docker/README.md +++ b/docker/README.md @@ -4,15 +4,25 @@ * Crowdsec concepts: https://docs.crowdsec.net/docs/concepts * Where to file issues: https://github.com/crowdsecurity/crowdsec - # What is Crowdsec Crowdsec - An open-source, lightweight agent to detect and respond to bad behaviours. It also automatically benefits from our global community-wide IP reputation database. # How to use this image +## Docker images available +crowdsec will use Alpine as default container. A debian container is also available with systemd for journalctl support. Simply add `-debian` to your tag to use this. Please be aware that debian containers are not available on all version, since the feature was implemented after the release of version 1.3.0 + ## Required configuration +### Journalctl +To use journalctl as log stream, eventually from the `DSN` environment variable, it's important that you mount the journal log from the host to the container it self. +This can be done by adding the following volume mount to your docker command: + +``` +-v /var/log/journal:/run/log/journal +``` + ### Logs ingestion and processing Collections are a good place to start: https://docs.crowdsec.net/docs/collections/intro @@ -127,6 +137,11 @@ Using binds rather than named volumes ([more explanation here](https://docs.dock * `CERT_FILE` - TLS Certificate file (default: `/etc/ssl/cert.pem`) : `-e CERT_FILE=""` * `KEY_FILE` - TLS Key file (default: `/etc/ssl/key.pem`) : `-e KEY_FILE=""` * `CUSTOM_HOSTNAME` - Custom hostname for local api (default: `localhost`) : `-e CUSTOM_HOSTNAME=""` +* `DISABLE_COLLECTIONS` - Collections to remove from the [hub](https://hub.crowdsec.net/browse/#collections), separated by space : `-e DISABLE_COLLECTIONS="crowdsecurity/linux crowdsecurity/nginx"` +* `DISABLE_PARSERS` - Parsers to remove from the [hub](https://hub.crowdsec.net/browse/#configurations), separated by space : `-e DISABLE_PARSERS="crowdsecurity/apache2-logs crowdsecurity/nginx-logs"` +* `DISABLE_SCENARIOS` - Scenarios to remove from the [hub](https://hub.crowdsec.net/browse/#configurations), separated by space : `-e DISABLE_SCENARIOS="crowdsecurity/http-bad-user-agent crowdsecurity/http-xss-probing"` +* `DISABLE_POSTOVERFLOWS` - Postoverflows to remove from the [hub](https://hub.crowdsec.net/browse/#configurations), separated by space : `-e DISABLE_POSTOVERFLOWS="crowdsecurity/cdn-whitelist crowdsecurity/seo-bots-whitelist"` +* `PLUGIN_DIR` - Directory for plugins (default: `/usr/local/lib/crowdsec/plugins/`) : `-e PLUGIN_DIR=""` ## Volumes @@ -137,7 +152,7 @@ Using binds rather than named volumes ([more explanation here](https://docs.dock ## File Locations * `/usr/local/bin/crowdsec` - Crowdsec binary - + * `/usr/local/bin/cscli` - Crowdsec CLI binary to interact with crowdsec # Find Us diff --git a/docker/docker_start.sh b/docker/docker_start.sh index f86b02004..7db38c097 100755 --- a/docker/docker_start.sh +++ b/docker/docker_start.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Set the crowdsec config file CS_CONFIG_FILE="/etc/crowdsec/config.yaml" @@ -10,6 +10,9 @@ fi CERT_FILE="${CERT_FILE:-/etc/ssl/cert.pem}" KEY_FILE="${KEY_FILE:-/etc/ssl/key.pem}" +# Plugins directory default +PLUGIN_DIR="${PLUGIN_DIR:-/usr/local/lib/crowdsec/plugins/}" + #Check & prestage databases if [ ! -e "/var/lib/data/GeoLite2-ASN.mmdb" ] && [ ! -e "/var/lib/data/GeoLite2-City.mmdb" ]; then mkdir -p /var/lib/crowdsec/data @@ -41,7 +44,7 @@ if [ "$DISABLE_AGENT" == "" ] ; then fi # Check if lapi needs to automatically register an agent -echo Check if lapi need to register automatically an agent +echo "Check if lapi need to register automatically an agent" if [ "$DISABLE_LOCAL_API" == "" ] && [ "$AGENT_USERNAME" != "" ] && [ "$AGENT_PASSWORD" != "" ] ; then if [ "$LOCAL_API_URL" != "" ] ; then cscli -c "$CS_CONFIG_FILE" machines add $AGENT_USERNAME --password $AGENT_PASSWORD --url $LOCAL_API_URL @@ -78,7 +81,11 @@ if [ "$USE_TLS" != "" ]; then yq -i eval '... comments=""' "$CS_CONFIG_FILE" fi -## Install collections, parsers & scenarios +if [ "$PLUGIN_DIR" != "/usr/local/lib/crowdsec/plugins/" ]; then + yq -i eval ".config_paths.plugin_dir = \"$PLUGIN_DIR\"" "$CS_CONFIG_FILE" +fi + +## Install collections, parsers, scenarios & postoverflows cscli -c "$CS_CONFIG_FILE" hub update cscli -c "$CS_CONFIG_FILE" collections upgrade crowdsecurity/linux || true cscli -c "$CS_CONFIG_FILE" parsers upgrade crowdsecurity/whitelists || true @@ -96,6 +103,20 @@ if [ "$POSTOVERFLOWS" != "" ]; then cscli -c "$CS_CONFIG_FILE" postoverflows install $POSTOVERFLOWS fi +## Remove collections, parsers, scenarios & postoverflows +if [ "$DISABLE_COLLECTIONS" != "" ]; then + cscli -c "$CS_CONFIG_FILE" collections remove $DISABLE_COLLECTIONS +fi +if [ "$DISABLE_PARSERS" != "" ]; then + cscli -c "$CS_CONFIG_FILE" parsers remove $DISABLE_PARSERS +fi +if [ "$DISABLE_SCENARIOS" != "" ]; then + cscli -c "$CS_CONFIG_FILE" scenarios remove $DISABLE_SCENARIOS +fi +if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then + cscli -c "$CS_CONFIG_FILE" postoverflows remove $DISABLE_POSTOVERFLOWS +fi + ARGS="" if [ "$CONFIG_FILE" != "" ]; then ARGS="-c $CONFIG_FILE"