beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

docker build flavors: slim, with-plugins, with-geoip, full (#1862)

Browse source
This commit is contained in:
mmetc 2022-11-08 12:28:57 +01:00 committed by GitHub
parent 502a3cf841
commit b0889d7751
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 166 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
.github/workflows/release_publish_docker-image.yml vendored
View file

@ -29,11 +29,14 @@ jobs:
VERSION=pr-${{ github.event.number }}
fi
TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}"
TAGS_SLIM="${DOCKER_IMAGE}:${VERSION}-slim"
if [[ ${{ github.event.action }} == released ]]; then
TAGS=$TAGS,${DOCKER_IMAGE}:latest,${GHCR_IMAGE}:latest
TAGS_SLIM=$TAGS,${DOCKER_IMAGE}:slim
fi
echo ::set-output name=version::${VERSION}
echo ::set-output name=tags::${TAGS}
echo ::set-output name=tags_slim::${TAGS_SLIM}
echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ')
-
name: Set up QEMU
@ -54,8 +57,25 @@ jobs:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
-
name: Build and push
name: Build and push slim image
uses: docker/build-push-action@v2
with:
context: .
file: ./Dockerfile
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.prep.outputs.tags_slim }}
build-args: |
BUILD_ENV=slim
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6,linux/386
labels: |
org.opencontainers.image.source=${{ github.event.repository.html_url }}
org.opencontainers.image.created=${{ steps.prep.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}
-
name: Build and push full image
uses: docker/build-push-action@v2
with:
context: .

42
Dockerfile
View file

@ -1,33 +1,51 @@
ARG BUILD_ENV=full
ARG GOVERSION=1.19
FROM golang:${GOVERSION}-alpine AS build
WORKDIR /go/src/crowdsec
# wizard.sh requires GNU coreutils
RUN apk add --no-cache git gcc libc-dev make bash gettext binutils-gold coreutils
COPY . .
RUN SYSTEM="docker" make release
RUN cd crowdsec-v* && ./wizard.sh --docker-mode && cd -
RUN cscli hub update && cscli collections install crowdsecurity/linux && cscli parsers install crowdsecurity/whitelists
FROM alpine:latest
# wizard.sh requires GNU coreutils
RUN apk add --no-cache git gcc libc-dev make bash gettext binutils-gold coreutils && \
SYSTEM="docker" make release && \
cd crowdsec-v* && \
./wizard.sh --docker-mode && \
cd - && \
cscli hub update && \
cscli collections install crowdsecurity/linux && \
cscli parsers install crowdsecurity/whitelists
FROM alpine:latest as build-slim
RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata yq bash && \
mkdir -p /staging/etc/crowdsec && \
mkdir -p /staging/var/lib/crowdsec
mkdir -p /staging/var/lib/crowdsec && \
mkdir -p /var/lib/crowdsec/data
COPY --from=build /etc/crowdsec /staging/etc/crowdsec
COPY --from=build /var/lib/crowdsec /staging/var/lib/crowdsec
COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
#Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
#The files are here for reference, as users will need to mount a new version to be actually able to use notifications
ENTRYPOINT /bin/bash docker_start.sh
FROM build-slim as build-plugins
# Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
# The files are here for reference, as users will need to mount a new version to be actually able to use notifications
COPY --from=build /go/src/crowdsec/plugins/notifications/email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins
ENTRYPOINT /bin/bash docker_start.sh
FROM build-slim as build-geoip
COPY --from=build /var/lib/crowdsec /staging/var/lib/crowdsec
FROM build-plugins as build-full
COPY --from=build /var/lib/crowdsec /staging/var/lib/crowdsec
FROM build-${BUILD_ENV}

48
Dockerfile.debian
View file

@ -1,23 +1,28 @@
ARG BUILD_ENV=full
ARG GOVERSION=1.19
FROM golang:${GOVERSION}-bullseye AS build
WORKDIR /go/src/crowdsec
# wizard.sh requires GNU coreutils
RUN apt-get update && apt-get install -y git gcc libc-dev make bash gettext binutils-gold coreutils tzdata python3 python3-pip
COPY . .
RUN SYSTEM="docker" make release
RUN cd crowdsec-v* && ./wizard.sh --docker-mode && cd -
RUN cscli hub update && cscli collections install crowdsecurity/linux && cscli parsers install crowdsecurity/whitelists
RUN GO111MODULE=on go get github.com/mikefarah/yq/v4
# wizard.sh requires GNU coreutils
RUN apt-get update && \
apt-get install -y git gcc libc-dev make bash gettext binutils-gold coreutils tzdata && \
SYSTEM="docker" make release && \
cd crowdsec-v* && \
./wizard.sh --docker-mode && \
cd - && \
cscli hub update && \
cscli collections install crowdsecurity/linux && \
cscli parsers install crowdsecurity/whitelists && \
go install github.com/mikefarah/yq/v4@latest
FROM debian:bullseye-slim
FROM debian:bullseye-slim as build-slim
RUN apt-get update
RUN apt-get install -y -q --install-recommends --no-install-suggests \
RUN apt-get update && \
apt-get install -y -q --install-recommends --no-install-suggests \
procps \
systemd \
iproute2 \
@ -25,22 +30,35 @@ RUN apt-get install -y -q --install-recommends --no-install-suggests \
bash \
tzdata && \
mkdir -p /staging/etc/crowdsec && \
mkdir -p /staging/var/lib/crowdsec
mkdir -p /staging/var/lib/crowdsec && \
mkdir -p /var/lib/crowdsec/data
COPY --from=build /go/bin/yq /usr/local/bin/yq
COPY --from=build /etc/crowdsec /staging/etc/crowdsec
COPY --from=build /var/lib/crowdsec /staging/var/lib/crowdsec
COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
RUN yq eval -i ".plugin_config.group = \"nogroup\"" /staging/etc/crowdsec/config.yaml
#Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
#The files are here for reference, as users will need to mount a new version to be actually able to use notifications
ENTRYPOINT /bin/bash docker_start.sh
FROM build-slim as build-plugins
# Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
# The files are here for reference, as users will need to mount a new version to be actually able to use notifications
COPY --from=build /go/src/crowdsec/plugins/notifications/email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins
ENTRYPOINT /bin/bash docker_start.sh
FROM build-slim as build-geoip
COPY --from=build /var/lib/crowdsec /staging/var/lib/crowdsec
FROM build-plugins as build-full
COPY --from=build /var/lib/crowdsec /staging/var/lib/crowdsec
FROM build-${BUILD_ENV}

60
docker/README.md
View file

@ -10,13 +10,53 @@ Crowdsec - An open-source, lightweight agent to detect and respond to bad behavi
# How to use this image
## Docker images available
crowdsec will use Alpine as default container. A debian container is also available with systemd for journalctl support. Simply add `-debian` to your tag to use this. Please be aware that debian containers are not available on all version, since the feature was implemented after the release of version 1.3.0
## Docker image versions
All the following versions are available on Docker Hub for 386, amd64, arm/v6, arm/v7, arm64.
### Alpine
- crowdsecurity/crowdsec:{version}
Recommended for production usage. Also available on GitHub (ghrc.io).
- crowdsecurity/crowdsec:latest
For development and testing.
since v1.4.2:
- crowdsecurity/crowdsec:slim
Reduced size by 60%, does not include notifier plugins nor the GeoIP database.
If you need these details on decisions, running `cscli hub upgrade` inside the
container downloads the GeoIP database at runtime.
### Debian (since v1.3.3)
- crowdsecurity/crowdsec:{version}-debian
- crowdsecurity/crowdsec:latest-debian
The debian version includes support for systemd and journalctl.
### Custom
You can build your own images with Dockerfile and Dockerfile-debian.
For example, if you want a Debian version without plugin notifiers:
```console
$ docker build -f Dockerfile.debian --build-arg=BUILD_ENV=slim
```
supported values for BUILD_ENV are: full, with-geoip, with-plugins, slim.
## Required configuration
### Journalctl (only for debian image)
To use journalctl (only for debian image) as log stream, eventually from the `DSN` environment variable, it's important that you mount the journal log from the host to the container it self.
To use journalctl (only with the debian image) as a log stream, eventually from the `DSN` environment variable, it's important to mount the journal log from the host to the container itself.
This can be done by adding the following volume mount to your docker command:
```
@ -77,12 +117,12 @@ docker run -d \
Check this full stack example using docker-compose: https://github.com/crowdsecurity/example-docker-compose
# How to extend this image
## Full configuration
The container is built with specific docker [configuration](https://github.com/crowdsecurity/crowdsec/blob/master/docker/config.yaml). If you need to change it, bind `/etc/crowdsec/config.yaml` to your local configuration file
The container is built with a specific docker [configuration](https://github.com/crowdsecurity/crowdsec/blob/master/docker/config.yaml). If you need to change it, bind `/etc/crowdsec/config.yaml` to your local configuration file
## Notifications
If you wish to use the [notification system](https://docs.crowdsec.net/docs/notification_plugins/intro), you will need to mount at least a custom `profiles.yaml` and a notification configuration to `/etc/crowdsec/notifications`
# Deployment use cases
Crowdsec is composed of an `agent` that parse logs and creates `alerts` that `local API` or `LAPI` transform into decisions. Both can run in the same process but also on separated containers as it makes sense in complex configurations to have agents on the same machines as the protected component and a LAPI that gather all signals from agents and communicate with the `central api`.
Crowdsec is composed of an `agent` that parses logs and creates `alerts` that `local API` or `LAPI` transform into decisions. Both can run in the same process but also on separated containers as it makes sense in complex configurations to have agents on the same machines as the protected component and a LAPI that gather all signals from agents and communicate with the `central api`.
## Register a new agent with LAPI
```shell
@ -90,7 +130,7 @@ docker exec -it crowdsec_lapi_container_name cscli machines add agent_user_name
```
## Run an agent connected to LAPI
Add following environment variables to your docker run command:
Add the following environment variables to your docker run command:
* `DISABLE_LOCAL_API=true`
* `AGENT_USERNAME="agent_user_name"` - agent_user_name previously registered with LAPI
* `AGENT_PASSWORD="agent_password"` - agent_password previously registered with LAPI
@ -98,7 +138,7 @@ Add following environment variables to your docker run command:
# Next steps
## Bouncers
Crowdsec being a detection component, remediation is implemented using `bouncers`. Each bouncer protect a specific component. Find out more:
Crowdsec being a detection component, remediation is implemented using `bouncers`. Each bouncer protects a specific component. Find out more:
https://hub.crowdsec.net/browse/#bouncers
@ -110,17 +150,17 @@ You can automatically register bouncers with the crowdsec container on startup u
To use environment variables, they should be in the format `BOUNCER_KEY_<name>=<key>`. e.g. `BOUNCER_KEY_nginx=mysecretkey12345`.
To use Docker secrets, the secret should be named `bouncer_key_<name>` with a content of `<key>`. e.g. `bouncer_key_nginx` with a content of `mysecretkey12345`.
To use Docker secrets, the secret should be named `bouncer_key_<name>` with a content of `<key>`. e.g. `bouncer_key_nginx` with content `mysecretkey12345`.
A bouncer key can be any string but we recommend an alphanumeric value to keep consistent with crowdsec-generated keys and avoid problems with escaping special characters.
## Console
We provide a web based interface to get more from Crowdsec: https://docs.crowdsec.net/docs/console
We provide a web-based interface to get more from Crowdsec: https://docs.crowdsec.net/docs/console
Subscribe here: https://app.crowdsec.net
# Caveats
Using binds rather than named volumes ([more explanation here](https://docs.docker.com/storage/volumes/)) results in more complexity as you'll have to bind relevant files one by one where with named volumes you can mount full configuration and data folders. On the other hand, named volumes are less straightforward to navigate.
Using binds rather than named volumes ([more explanation here](https://docs.docker.com/storage/volumes/)) results in more complexity as you'll have to bind relevant files one by one whereas with named volumes you can mount full configuration and data folders. On the other hand, named volumes are less straightforward to navigate.
# Reference
## Environment Variables

47
docker/docker_start.sh
View file

@ -13,11 +13,17 @@ KEY_FILE="${KEY_FILE:-/etc/ssl/key.pem}"
# Plugins directory default
PLUGIN_DIR="${PLUGIN_DIR:-/usr/local/lib/crowdsec/plugins/}"
#Check & prestage databases
if [ ! -e "/var/lib/crowdsec/data/GeoLite2-ASN.mmdb" ] && [ ! -e "/var/lib/crowdsec/data/GeoLite2-City.mmdb" ]; then
mkdir -p /var/lib/crowdsec/data
cp /staging/var/lib/crowdsec/data/*.mmdb /var/lib/crowdsec/data/
fi
# Check & prestage databases
for geodb in GeoLite2-ASN.mmdb GeoLite2-City.mmdb; do
# We keep the pre-populated geoib databases in /staging instead of /var,
# because if the data directory is bind-mounted from the host, it will be
# empty and the files will be out of reach, requiring a runtime download.
# We link to them to save about 80Mb compared to cp/mv.
if [ ! -e "/var/lib/crowdsec/data/$geodb" ] && [ -e "/staging/var/lib/crowdsec/data/$geodb" ]; then
mkdir -p /var/lib/crowdsec/data
ln -s "/staging/var/lib/crowdsec/data/$geodb" /var/lib/crowdsec/data/
fi
done
#Check & prestage /etc/crowdsec
if [ ! -e "/etc/crowdsec/local_api_credentials.yaml" ] && [ ! -e "/etc/crowdsec/config.yaml" ]; then
@ -28,18 +34,18 @@ fi
# regenerate local agent credentials (ignore if agent is disabled)
if [ "$DISABLE_AGENT" == "" ] ; then
echo "Regenerate local agent credentials"
cscli -c "$CS_CONFIG_FILE" machines delete ${CUSTOM_HOSTNAME:-localhost}
cscli -c "$CS_CONFIG_FILE" machines delete "${CUSTOM_HOSTNAME:-localhost}"
if [ "$LOCAL_API_URL" != "" ] ; then
cscli -c "$CS_CONFIG_FILE" machines add ${CUSTOM_HOSTNAME:-localhost} --auto --url $LOCAL_API_URL
cscli -c "$CS_CONFIG_FILE" machines add "${CUSTOM_HOSTNAME:-localhost}" --auto --url "$LOCAL_API_URL"
else
cscli -c "$CS_CONFIG_FILE" machines add ${CUSTOM_HOSTNAME:-localhost} --auto
cscli -c "$CS_CONFIG_FILE" machines add "${CUSTOM_HOSTNAME:-localhost}" --auto
fi
if [ "$AGENT_USERNAME" != "" ] && [ "$AGENT_PASSWORD" != "" ] && [ "$LOCAL_API_URL" != "" ] ; then
echo "set up lapi credentials for agent"
CONFIG_PATH=$(yq eval '.api.client.credentials_path' "$CS_CONFIG_FILE" )
echo "url: $LOCAL_API_URL" > $CONFIG_PATH
echo "login: $AGENT_USERNAME" >> $CONFIG_PATH
echo "password: $AGENT_PASSWORD" >> $CONFIG_PATH
echo "url: $LOCAL_API_URL" > "$CONFIG_PATH"
echo "login: $AGENT_USERNAME" >> "$CONFIG_PATH"
echo "password: $AGENT_PASSWORD" >> "$CONFIG_PATH"
fi
fi
@ -47,9 +53,9 @@ fi
echo "Check if lapi need to register automatically an agent"
if [ "$DISABLE_LOCAL_API" == "" ] && [ "$AGENT_USERNAME" != "" ] && [ "$AGENT_PASSWORD" != "" ] ; then
if [ "$LOCAL_API_URL" != "" ] ; then
cscli -c "$CS_CONFIG_FILE" machines add $AGENT_USERNAME --password $AGENT_PASSWORD --url $LOCAL_API_URL
cscli -c "$CS_CONFIG_FILE" machines add "$AGENT_USERNAME" --password "$AGENT_PASSWORD" --url "$LOCAL_API_URL"
else
cscli -c "$CS_CONFIG_FILE" machines add $AGENT_USERNAME --password $AGENT_PASSWORD
cscli -c "$CS_CONFIG_FILE" machines add "$AGENT_USERNAME" --password "$AGENT_PASSWORD"
fi
echo "Agent registered to lapi"
fi
@ -72,12 +78,14 @@ if [ "${DISABLE_ONLINE_API,,}" != "true" ] && [ "$ENROLL_KEY" != "" ] ; then
enroll_args="--name $ENROLL_INSTANCE_NAME"
fi
if [ "$ENROLL_TAGS" != "" ] ; then
#shellcheck disable=SC2086
for tag in ${ENROLL_TAGS}
do
enroll_args="$enroll_args --tags $tag"
done
fi
cscli console enroll $enroll_args $ENROLL_KEY
#shellcheck disable=SC2086
cscli console enroll $enroll_args "$ENROLL_KEY"
fi
# crowdsec sqlite database permissions
@ -85,7 +93,7 @@ if [ "$GID" != "" ]; then
IS_SQLITE=$(yq eval '.db_config.type == "sqlite"' "$CS_CONFIG_FILE")
DB_PATH=$(yq eval '.db_config.db_path' "$CS_CONFIG_FILE")
if [ "$IS_SQLITE" == "true" ]; then
chown :$GID $DB_PATH
chown ":$GID" "$DB_PATH"
echo "sqlite database permissions updated"
fi
fi
@ -106,29 +114,37 @@ cscli -c "$CS_CONFIG_FILE" collections upgrade crowdsecurity/linux || true
cscli -c "$CS_CONFIG_FILE" parsers upgrade crowdsecurity/whitelists || true
cscli -c "$CS_CONFIG_FILE" parsers install crowdsecurity/docker-logs || true
if [ "$COLLECTIONS" != "" ]; then
#shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" collections install $COLLECTIONS
fi
if [ "$PARSERS" != "" ]; then
#shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" parsers install $PARSERS
fi
if [ "$SCENARIOS" != "" ]; then
#shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" scenarios install $SCENARIOS
fi
if [ "$POSTOVERFLOWS" != "" ]; then
#shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" postoverflows install $POSTOVERFLOWS
fi
## Remove collections, parsers, scenarios & postoverflows
if [ "$DISABLE_COLLECTIONS" != "" ]; then
#shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" collections remove $DISABLE_COLLECTIONS
fi
if [ "$DISABLE_PARSERS" != "" ]; then
#shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" parsers remove $DISABLE_PARSERS
fi
if [ "$DISABLE_SCENARIOS" != "" ]; then
#shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" scenarios remove $DISABLE_SCENARIOS
fi
if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
#shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" postoverflows remove $DISABLE_POSTOVERFLOWS
fi
@ -192,4 +208,5 @@ if [ "${LEVEL_INFO,,}" == "true" ]; then
ARGS="$ARGS -info"
fi
#shellcheck disable=SC2086
exec crowdsec $ARGS