beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Docker refactoring, tls setup (#1869)

Browse source
This commit is contained in:
mmetc 2022-11-28 10:35:12 +01:00 committed by GitHub
parent c5079ac15e
commit fde9640364
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 377 additions and 156 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
Dockerfile
View file

@ -9,7 +9,7 @@ COPY . .
# wizard.sh requires GNU coreutils # wizard.sh requires GNU coreutils
RUN apk add --no-cache git gcc libc-dev make bash gettext binutils-gold coreutils && \ RUN apk add --no-cache git gcc libc-dev make bash gettext binutils-gold coreutils && \
SYSTEM="docker" make release && \ SYSTEM="docker" make clean release && \
cd crowdsec-v* && \ cd crowdsec-v* && \
./wizard.sh --docker-mode && \ ./wizard.sh --docker-mode && \
cd - && \ cd - && \
@ -23,15 +23,72 @@ RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/co
mkdir -p /staging/etc/crowdsec && \ mkdir -p /staging/etc/crowdsec && \
mkdir -p /staging/var/lib/crowdsec && \ mkdir -p /staging/var/lib/crowdsec && \
mkdir -p /var/lib/crowdsec/data mkdir -p /var/lib/crowdsec/data
COPY --from=build /etc/crowdsec /staging/etc/crowdsec COPY --from=build /etc/crowdsec /staging/etc/crowdsec
COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
COPY --from=build /go/src/crowdsec/docker/docker_start.sh / COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
ENV CONFIG_FILE=/etc/crowdsec/config.yaml
ENV LOCAL_API_URL=http://0.0.0.0:8080/
ENV CUSTOM_HOSTNAME=localhost
ENV PLUGIN_DIR=/usr/local/lib/crowdsec/plugins/
ENV DISABLE_AGENT=false
ENV DISABLE_LOCAL_API=false
ENV DISABLE_ONLINE_API=false
ENV DSN=
ENV TYPE=
ENV TEST_MODE=false
# register to app.crowdsec.net
ENV ENROLL_INSTANCE_NAME=
ENV ENROLL_KEY=
ENV ENROLL_TAGS=
# log verbosity
ENV LEVEL_TRACE=false
ENV LEVEL_DEBUG=false
ENV LEVEL_INFO=true
# TLS setup ----------------------------------- #
ENV AGENT_USERNAME=
ENV AGENT_PASSWORD=
# TLS setup ----------------------------------- #
ENV USE_TLS=false
ENV CA_CERT_PATH=
ENV CERT_FILE=/etc/ssl/cert.pem
ENV KEY_FILE=/etc/ssl/key.pem
# comma-separated list of allowed OU values for TLS bouncer certificates
ENV BOUNCERS_ALLOWED_OU=bouncer-ou
# comma-separated list of allowed OU values for TLS agent certificates
ENV AGENTS_ALLOWED_OU=agent-ou
# Install the following hub items --------------#
ENV COLLECTIONS=
ENV PARSERS=
ENV SCENARIOS=
ENV POSTOVERFLOWS=
# Uninstall the following hub items ------------#
ENV DISABLE_COLLECTIONS=
ENV DISABLE_PARSERS=
ENV DISABLE_SCENARIOS=
ENV DISABLE_POSTOVERFLOWS=
ENV METRICS_PORT=6060
ENTRYPOINT /bin/bash docker_start.sh ENTRYPOINT /bin/bash docker_start.sh
FROM build-slim as build-plugins FROM build-slim as build-plugins
# Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp # Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
# The files are here for reference, as users will need to mount a new version to be actually able to use notifications # The files are here for reference, as users will need to mount a new version to be actually able to use notifications
COPY --from=build /go/src/crowdsec/plugins/notifications/email/email.yaml /staging/etc/crowdsec/notifications/email.yaml COPY --from=build /go/src/crowdsec/plugins/notifications/email/email.yaml /staging/etc/crowdsec/notifications/email.yaml

56
Dockerfile.debian
View file

@ -1,3 +1,4 @@
# vim: set ft=dockerfile:
ARG BUILD_ENV=full ARG BUILD_ENV=full
ARG GOVERSION=1.19 ARG GOVERSION=1.19
@ -41,6 +42,61 @@ COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
RUN yq eval -i ".plugin_config.group = \"nogroup\"" /staging/etc/crowdsec/config.yaml RUN yq eval -i ".plugin_config.group = \"nogroup\"" /staging/etc/crowdsec/config.yaml
ENV CONFIG_FILE=/etc/crowdsec/config.yaml
ENV LOCAL_API_URL=http://0.0.0.0:8080/
ENV CUSTOM_HOSTNAME=localhost
ENV PLUGIN_DIR=/usr/local/lib/crowdsec/plugins/
ENV DISABLE_AGENT=false
ENV DISABLE_LOCAL_API=false
ENV DISABLE_ONLINE_API=false
ENV DSN=
ENV TYPE=
ENV TEST_MODE=false
# register to app.crowdsec.net
ENV ENROLL_INSTANCE_NAME=
ENV ENROLL_KEY=
ENV ENROLL_TAGS=
# log verbosity
ENV LEVEL_TRACE=false
ENV LEVEL_DEBUG=false
ENV LEVEL_INFO=true
# TLS setup ----------------------------------- #
ENV AGENT_USERNAME=
ENV AGENT_PASSWORD=
# TLS setup ----------------------------------- #
ENV USE_TLS=false
ENV CA_CERT_PATH=
ENV CERT_FILE=/etc/ssl/cert.pem
ENV KEY_FILE=/etc/ssl/key.pem
# comma-separated list of allowed OU values for TLS bouncer certificates
ENV BOUNCERS_ALLOWED_OU=bouncer-ou
# comma-separated list of allowed OU values for TLS agent certificates
ENV AGENTS_ALLOWED_OU=agent-ou
# Install the following hub items --------------#
ENV COLLECTIONS=
ENV PARSERS=
ENV SCENARIOS=
ENV POSTOVERFLOWS=
# Uninstall the following hub items ------------#
ENV DISABLE_COLLECTIONS=
ENV DISABLE_PARSERS=
ENV DISABLE_SCENARIOS=
ENV DISABLE_POSTOVERFLOWS=
ENV METRICS_PORT=6060
ENTRYPOINT /bin/bash docker_start.sh ENTRYPOINT /bin/bash docker_start.sh
FROM build-slim as build-plugins FROM build-slim as build-plugins

144
docker/README.md
View file

@ -10,23 +10,24 @@ Crowdsec - An open-source, lightweight agent to detect and respond to bad behavi
# How to use this image # How to use this image
## Docker image versions ## Image flavors
All the following versions are available on Docker Hub for 386, amd64, arm/v6, arm/v7, arm64. All the following images are available on Docker Hub for the architectures
386, amd64, arm/v6, arm/v7, arm64.
### Alpine ### Alpine
- crowdsecurity/crowdsec:{version} - `crowdsecurity/crowdsec:{version}`
Recommended for production usage. Also available on GitHub (ghrc.io). Recommended for production usage. Also available on GitHub (ghrc.io).
- crowdsecurity/crowdsec:latest - `crowdsecurity/crowdsec:latest`
For development and testing. For development and testing.
since v1.4.2: since v1.4.2:
- crowdsecurity/crowdsec:slim - `crowdsecurity/crowdsec:slim`
Reduced size by 60%, does not include notifier plugins nor the GeoIP database. Reduced size by 60%, does not include notifier plugins nor the GeoIP database.
If you need these details on decisions, running `cscli hub upgrade` inside the If you need these details on decisions, running `cscli hub upgrade` inside the
@ -35,14 +36,14 @@ container downloads the GeoIP database at runtime.
### Debian (since v1.3.3) ### Debian (since v1.3.3)
- crowdsecurity/crowdsec:{version}-debian - `crowdsecurity/crowdsec:{version}-debian`
- crowdsecurity/crowdsec:latest-debian - `crowdsecurity/crowdsec:latest-debian`
The debian version includes support for systemd and journalctl. The debian version includes support for systemd and journalctl.
### Custom ### Custom
You can build your own images with Dockerfile and Dockerfile-debian. You can build your images with Dockerfile and Dockerfile-debian.
For example, if you want a Debian version without plugin notifiers: For example, if you want a Debian version without plugin notifiers:
@ -56,8 +57,8 @@ supported values for BUILD_ENV are: full, with-geoip, with-plugins, slim.
## Required configuration ## Required configuration
### Journalctl (only for debian image) ### Journalctl (only for debian image)
To use journalctl (only with the debian image) as a log stream, eventually from the `DSN` environment variable, it's important to mount the journal log from the host to the container itself. To use journalctl as a log stream, with or without the `DSN` environment variable, it's important to mount the journal log from the host to the container itself.
This can be done by adding the following volume mount to your docker command: This can be done by adding the following volume mount to the docker command:
``` ```
-v /var/log/journal:/run/log/journal -v /var/log/journal:/run/log/journal
@ -66,14 +67,14 @@ This can be done by adding the following volume mount to your docker command:
### Logs ingestion and processing ### Logs ingestion and processing
Collections are a good place to start: https://docs.crowdsec.net/docs/collections/intro Collections are a good place to start: https://docs.crowdsec.net/docs/collections/intro
Find collections|scenarios|parsers|postoverflows in the hub: https://hub.crowdsec.net Find collections, scenarios, parsers and postoverflows in the hub: https://hub.crowdsec.net
* Specify collections|scenarios|parsers/postoverflows to install via the environment variables (by default [`crowdsecurity/linux`](https://hub.crowdsec.net/author/crowdsecurity/collections/linux) is installed) * Specify collections | scenarios | parsers | postoverflows to install via the environment variables (by default [`crowdsecurity/linux`](https://hub.crowdsec.net/author/crowdsecurity/collections/linux) is installed)
* Mount volumes to specify your log files that should be ingested by crowdsec * Mount volumes to specify your log files that should be ingested by crowdsec
### Acquisition ### Acquisition
`/etc/crowdsec/acquis.yaml` maps logs to provided parsers. Find out more here: https://docs.crowdsec.net/docs/concepts/#acquisition `/etc/crowdsec/acquis.yaml` maps logs to the provided parsers. Find out more here: https://docs.crowdsec.net/docs/concepts/#acquisition
acquis.yaml example: acquis.yaml example:
```shell ```shell
@ -88,14 +89,15 @@ labels:
type: apache2 type: apache2
``` ```
`labels.type`: use `syslog` if logs origin is `syslog`, checkout collection's documentation for the relevant type otherwise. `labels.type`: use `syslog` if the logs come from syslog, otherwise check the collection's documentation for the relevant type.
## Recommended configuration ## Recommended configuration
### Volumes ### Volumes
We strongly suggest to mount **named volumes** for Crowdsec configuration and database to avoid credentials and decisions loss in case of container's destruction and recreation, version update, etc. We strongly suggest mounting **named volumes** for Crowdsec configuration and database to avoid credentials and decisions loss in case of container destruction and recreation, version update, etc.
* Credentials and configuration: `/etc/crowdsec` * Credentials and configuration: `/etc/crowdsec`
* Database when using default SQLite: `/var/lib/crowdsec/data` * Database when using SQLite (default): `/var/lib/crowdsec/data`
## Start a Crowdsec instance ## Start a Crowdsec instance
@ -114,15 +116,26 @@ docker run -d \
## ... or docker-compose ## ... or docker-compose
Check this full stack example using docker-compose: https://github.com/crowdsecurity/example-docker-compose Check this full-stack example using docker-compose: https://github.com/crowdsecurity/example-docker-compose
# How to extend this image # How to extend this image
## Full configuration ## Full configuration
The container is built with a specific docker [configuration](https://github.com/crowdsecurity/crowdsec/blob/master/docker/config.yaml). If you need to change it, bind `/etc/crowdsec/config.yaml` to your local configuration file The container is built with a specific docker
[configuration](https://github.com/crowdsecurity/crowdsec/blob/master/docker/config.yaml).
If you need to change it and the docker variables (see below) are not enough,
you can bind `/etc/crowdsec/config.yaml` to your a configuration file.
## Notifications ## Notifications
If you wish to use the [notification system](https://docs.crowdsec.net/docs/notification_plugins/intro), you will need to mount at least a custom `profiles.yaml` and a notification configuration to `/etc/crowdsec/notifications` If you wish to use the [notification system](https://docs.crowdsec.net/docs/notification_plugins/intro), you will need to mount at least a custom `profiles.yaml` and a notification configuration to `/etc/crowdsec/notifications`
# Deployment use cases # Deployment use cases
Crowdsec is composed of an `agent` that parses logs and creates `alerts` that `local API` or `LAPI` transform into decisions. Both can run in the same process but also on separated containers as it makes sense in complex configurations to have agents on the same machines as the protected component and a LAPI that gather all signals from agents and communicate with the `central api`. Crowdsec is composed of an `agent` that parses logs and creates `alerts`, and a
`local API (LAPI)` that transforms these alerts into decisions. Both functions
are provided by the same executables, so the agent and the LAPI can run in the
same or separate containers. In complex configurations, it makes sense to have
agents on each machine that runs the protected applications, and a LAPI that
gathers all signals from agents and communicates with the `central API`.
## Register a new agent with LAPI ## Register a new agent with LAPI
```shell ```shell
@ -130,15 +143,19 @@ docker exec -it crowdsec_lapi_container_name cscli machines add agent_user_name
``` ```
## Run an agent connected to LAPI ## Run an agent connected to LAPI
Add the following environment variables to your docker run command:
Add the following environment variables to the docker run command:
* `DISABLE_LOCAL_API=true` * `DISABLE_LOCAL_API=true`
* `AGENT_USERNAME="agent_user_name"` - agent_user_name previously registered with LAPI * `AGENT_USERNAME="agent_user_name"` - agent_user_name previously registered with LAPI
* `AGENT_PASSWORD="agent_password"` - agent_password previously registered with LAPI * `AGENT_PASSWORD="agent_password"` - agent_password previously registered with LAPI
* `LOCAL_API_URL="http://LAPI_host:LAPI_port"` * `LOCAL_API_URL="http://LAPI_host:LAPI_port"`
# Next steps # Next steps
## Bouncers ## Bouncers
Crowdsec being a detection component, remediation is implemented using `bouncers`. Each bouncer protects a specific component. Find out more:
Crowdsec being a detection component, the remediation is implemented using `bouncers`. Each bouncer protects a specific component. Find out more:
https://hub.crowdsec.net/browse/#bouncers https://hub.crowdsec.net/browse/#bouncers
@ -146,13 +163,13 @@ https://docs.crowdsec.net/docs/user_guides/bouncers_configuration/
### Automatic Bouncer Registration ### Automatic Bouncer Registration
You can automatically register bouncers with the crowdsec container on startup using environment variables or Docker secrets. You cannot use this process to update an existing bouncer without first deleting it. You can automatically register bouncers with the crowdsec container at startup, using environment variables or Docker secrets. You cannot use this process to update an existing bouncer without first deleting it.
To use environment variables, they should be in the format `BOUNCER_KEY_<name>=<key>`. e.g. `BOUNCER_KEY_nginx=mysecretkey12345`. To use environment variables, they should be in the format `BOUNCER_KEY_<name>=<key>`. e.g. `BOUNCER_KEY_nginx=mysecretkey12345`.
To use Docker secrets, the secret should be named `bouncer_key_<name>` with a content of `<key>`. e.g. `bouncer_key_nginx` with content `mysecretkey12345`. To use Docker secrets, the secret should be named `bouncer_key_<name>` with a content of `<key>`. e.g. `bouncer_key_nginx` with content `mysecretkey12345`.
A bouncer key can be any string but we recommend an alphanumeric value to keep consistent with crowdsec-generated keys and avoid problems with escaping special characters. A bouncer key can be any string but we recommend an alphanumeric value for consistency with crowdsec-generated keys and avoid problems with escaping special characters.
## Console ## Console
We provide a web-based interface to get more from Crowdsec: https://docs.crowdsec.net/docs/console We provide a web-based interface to get more from Crowdsec: https://docs.crowdsec.net/docs/console
@ -160,42 +177,59 @@ We provide a web-based interface to get more from Crowdsec: https://docs.crowdse
Subscribe here: https://app.crowdsec.net Subscribe here: https://app.crowdsec.net
# Caveats # Caveats
Using binds rather than named volumes ([more explanation here](https://docs.docker.com/storage/volumes/)) results in more complexity as you'll have to bind relevant files one by one whereas with named volumes you can mount full configuration and data folders. On the other hand, named volumes are less straightforward to navigate.
Using binds rather than named volumes ([complete explanation here](https://docs.docker.com/storage/volumes/)) results in more complexity as you'll have to bind the relevant files one by one whereas with named volumes you can mount full configuration and data folders. On the other hand, named volumes are less straightforward to navigate.
# Reference # Reference
## Environment Variables ## Environment Variables
* `COLLECTIONS` - Collections to install from the [hub](https://hub.crowdsec.net/browse/#collections), separated by space : `-e COLLECTIONS="crowdsecurity/linux crowdsecurity/apache2"` | Variable | Default | Description |
* `SCENARIOS` - Scenarios to install from the [hub](https://hub.crowdsec.net/browse/#configurations), separated by space : `-e SCENARIOS="crowdsecurity/http-bad-user-agent crowdsecurity/http-xss-probing"` | ----------------------- | ------------------------- | ----------- |
* `PARSERS` - Parsers to install from the [hub](https://hub.crowdsec.net/browse/#configurations), separated by space : `-e PARSERS="crowdsecurity/http-logs crowdsecurity/modsecurity"` | `CONFIG_FILE` | `/etc/crowdsec/config.yaml` | Configuration file location |
* `POSTOVERFLOWS` - Postoverflows to install from the [hub](https://hub.crowdsec.net/browse/#configurations), separated by space : `-e POSTOVERFLOWS="crowdsecurity/cdn-whitelist"` | `DSN` | | Process a single source in time-machine: `-e DSN="file:///var/log/toto.log"` or `-e DSN="cloudwatch:///your/group/path:stream_name?profile=dev&backlog=16h"` or `-e DSN="journalctl://filters=_SYSTEMD_UNIT=ssh.service"` |
* `CONFIG_FILE` - Configuration file (default: `/etc/crowdsec/config.yaml`) : `-e CONFIG_FILE="<config_path>"` | `TYPE` | | [`Labels.type`](https://docs.crowdsec.net/Crowdsec/v1/references/acquisition/) for file in time-machine: `-e TYPE="<type>"` |
* `DSN` - Process a single source in time-machine : `-e DSN="file:///var/log/toto.log"` or `-e DSN="cloudwatch:///your/group/path:stream_name?profile=dev&backlog=16h"` or `-e DSN="journalctl://filters=_SYSTEMD_UNIT=ssh.service"` | `TEST_MODE` | false | Don't run the service, only test the configuration: `-e TEST_MODE=true` |
* `TYPE` - [`Labels.type`](https://docs.crowdsec.net/Crowdsec/v1/references/acquisition/) for file in time-machine : `-e TYPE="<type>"` | `TZ` | | Set the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) to ensure the logs have a local timestamp. |
* `TEST_MODE` - Only test configs (default: `false`) : `-e TEST_MODE="<true|false>"` | `LOCAL_API_URL` | `http://0.0.0.0:8080` | The LAPI URL, you need to change this when `DISABLE_LOCAL_API` is true: `-e LOCAL_API_URL="http://lapi-address:8080"` |
* `TZ` - Set the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) to ensure logs have a local timestamp. | `DISABLE_AGENT` | false | Disable the agent, run a LAPI-only container |
* `DISABLE_AGENT` - Only test configs (default: `false`) : `-e DISABLE_AGENT="<true|false>"` | `DISABLE_LOCAL_API` | false | Disable LAPI, run an agent-only container |
* `DISABLE_LOCAL_API` - Disable local API (default: `false`) : `-e DISABLE_LOCAL_API="<true|false>"` | `DISABLE_ONLINE_API` | false | Disable online API registration for signal sharing |
* `AGENT_USERNAME` - Agent username (to register if is LAPI or to use if it's an agent) : `-e AGENT_USERNAME="machine_id"` | `CUSTOM_HOSTNAME` | localhost | Custom hostname for LAPI registration (with agent and LAPI on the same container) |
* `AGENT_PASSWORD` - Agent password (to register if is LAPI or to use if it's an agent) : `-e AGENT_PASSWORD="machine_password"` | `PLUGIN_DIR` | `/usr/local/lib/crowdsec/plugins/` | Directory for plugins: `-e PLUGIN_DIR="<path>"` |
* `LOCAL_API_URL` - To specify when an agent needs to connect to a LAPI crowdsec (To use only when `DISABLE_LOCAL_API` is set to `true`) : `-e LOCAL_API_URL="http://lapi-address:8080"` | `BOUNCER_KEY_<name>` | | Register a bouncer with the name `<name>` and a key equal to the value of the environment variable. |
* `DISABLE_ONLINE_API` - Disable Online API registration for signal sharing (default: `false`) : `-e DISABLE_ONLINE_API="<true|false>"` | `METRICS_PORT` | 6060 | Port to expose Prometheus metrics |
* `LEVEL_TRACE` - Trace-level (VERY verbose) on stdout (default: `false`) : `-e LEVEL_TRACE="<true|false>"` | | | |
* `LEVEL_DEBUG` - Debug-level on stdout (default: `false`) : `-e LEVEL_DEBUG="<true|false>"` | __Console__ | | |
* `LEVEL_INFO` - Info-level on stdout (default: `false`) : `-e LEVEL_INFO="<true|false>"` | `ENROLL_KEY` | | Enroll key retrieved from [the console](https://app.crowdsec.net/) to enroll the instance. |
* `USE_TLS` - Enable TLS on the API Server (default: `false`) : `-e USE_TLS="<true|false>"` | `ENROLL_INSTANCE_NAME` | | To set an instance name and see it on [the console](https://app.crowdsec.net/) |
* `CERT_FILE` - TLS Certificate file (default: `/etc/ssl/cert.pem`) : `-e CERT_FILE="<file_path>"` | `ENROLL_TAGS` | | Tags of the enrolled instance, for search and filter |
* `KEY_FILE` - TLS Key file (default: `/etc/ssl/key.pem`) : `-e KEY_FILE="<file_path>"` | | | |
* `CUSTOM_HOSTNAME` - Custom hostname for local api (default: `localhost`) : `-e CUSTOM_HOSTNAME="<hostname>"` | __Password Auth__ | | |
* `DISABLE_COLLECTIONS` - Collections to remove from the [hub](https://hub.crowdsec.net/browse/#collections), separated by space : `-e DISABLE_COLLECTIONS="crowdsecurity/linux crowdsecurity/nginx"` | `AGENT_USERNAME` | | Agent username (to register if is LAPI or to use if it's an agent): `-e AGENT_USERNAME="machine_id"` |
* `DISABLE_PARSERS` - Parsers to remove from the [hub](https://hub.crowdsec.net/browse/#configurations), separated by space : `-e DISABLE_PARSERS="crowdsecurity/apache2-logs crowdsecurity/nginx-logs"` | `AGENT_PASSWORD` | | Agent password (to register if is LAPI or to use if it's an agent): `-e AGENT_PASSWORD="machine_password"` |
* `DISABLE_SCENARIOS` - Scenarios to remove from the [hub](https://hub.crowdsec.net/browse/#configurations), separated by space : `-e DISABLE_SCENARIOS="crowdsecurity/http-bad-user-agent crowdsecurity/http-xss-probing"` | | | |
* `DISABLE_POSTOVERFLOWS` - Postoverflows to remove from the [hub](https://hub.crowdsec.net/browse/#configurations), separated by space : `-e DISABLE_POSTOVERFLOWS="crowdsecurity/cdn-whitelist crowdsecurity/seo-bots-whitelist"` | __TLS Auth/encryption | | |
* `PLUGIN_DIR` - Directory for plugins (default: `/usr/local/lib/crowdsec/plugins/`) : `-e PLUGIN_DIR="<path>"` | `USE_TLS` | false | Enable TLS on the LAPI |
* `BOUNCER_KEY_<name>` - Register a bouncer with the name `<name>` and a key equal to the value of the environment variable. | `CERT_FILE` | /etc/ssl/cert.pem | TLS Certificate path |
* `ENROLL_KEY` - Enroll key retrieved from [the console](https://app.crowdsec.net/) to enroll the instance. | `KEY_FILE` | /etc/ssl/key.pem | TLS Key path |
* `ENROLL_INSTANCE_NAME` - To set an instance name and see it on [the console](https://app.crowdsec.net/). | `CACERT_FILE` | | CA certificate |
* `ENROLL_TAGS` - To set tags when enrolling an instance and use them for search and filtering on [the console](https://app.crowdsec.net/) | `AGENTS_ALLOWED_OU` | agent-ou | OU values allowed for agents, separated by comma |
| `BOUNCERS_ALLOWED_OU` | bouncer-ou | OU values allowed for bouncers, separated by comma |
| | | |
| __Hub management__ | | |
| `COLLECTIONS` | | Collections to install, separated by space: `-e COLLECTIONS="crowdsecurity/linux crowdsecurity/apache2"` |
| `SCENARIOS` | | Scenarios to install, separated by space |
| `PARSERS` | | Parsers to install, separated by space |
| `POSTOVERFLOWS` | | Postoverflows to install, separated by space |
| `DISABLE_COLLECTIONS` | | Collections to remove, separated by space: `-e DISABLE_COLLECTIONS="crowdsecurity/linux crowdsecurity/nginx"` |
| `DISABLE_PARSERS` | | Parsers to remove, separated by space |
| `DISABLE_SCENARIOS` | | Scenarios to remove, separated by space |
| `DISABLE_POSTOVERFLOWS` | | Postoverflows to remove, separated by space |
| | | |
| __Log verbosity__ | | |
| `LEVEL_INFO` | false | Force INFO level for the container log |
| `LEVEL_DEBUG` | false | Force DEBUG level for the container log |
| `LEVEL_TRACE` | false | Force TRACE level (VERY verbose) for the container log |
## Volumes ## Volumes

268
docker/docker_start.sh
View file

@ -1,21 +1,60 @@
#!/bin/bash #!/bin/bash
# Set the crowdsec config file # shellcheck disable=SC2292 # allow [ test ] syntax
CS_CONFIG_FILE="/etc/crowdsec/config.yaml" # shellcheck disable=SC2310 # allow "if function..." syntax with -e
if [ "$CONFIG_FILE" != "" ]; then
CS_CONFIG_FILE="$CONFIG_FILE"
fi
# TLS defaults set -e
CERT_FILE="${CERT_FILE:-/etc/ssl/cert.pem}" shopt -s inherit_errexit
KEY_FILE="${KEY_FILE:-/etc/ssl/key.pem}"
# Plugins directory default #- HELPER FUNCTIONS ----------------#
PLUGIN_DIR="${PLUGIN_DIR:-/usr/local/lib/crowdsec/plugins/}"
# Check & prestage databases istrue() {
case "$(echo "$1" | tr '[:upper:]' '[:lower:]')" in
true) return 0 ;;
*) return 1 ;;
esac
}
isfalse() {
if istrue "$1"; then
return 1
else
return 0
fi
}
# generate a yaml list from a comma-separated string of values
csv2yaml() {
[ -z "$1" ] && return
echo "$1" | sed 's/,/\n- /g;s/^/- /g'
}
# wrap cscli with the correct config file location
cscli() {
command cscli -c "$CONFIG_FILE" "$@"
}
conf_get() {
if [ $# -ge 2 ]; then
yq e "$1" "$2"
else
yq e "$1" "$CONFIG_FILE"
fi
}
conf_set() {
if [ $# -ge 2 ]; then
yq e "$1" -i "$2"
else
yq e "$1" -i "$CONFIG_FILE"
fi
}
#-----------------------------------#
# Check and prestage databases
for geodb in GeoLite2-ASN.mmdb GeoLite2-City.mmdb; do for geodb in GeoLite2-ASN.mmdb GeoLite2-City.mmdb; do
# We keep the pre-populated geoib databases in /staging instead of /var, # We keep the pre-populated geoip databases in /staging instead of /var,
# because if the data directory is bind-mounted from the host, it will be # because if the data directory is bind-mounted from the host, it will be
# empty and the files will be out of reach, requiring a runtime download. # empty and the files will be out of reach, requiring a runtime download.
# We link to them to save about 80Mb compared to cp/mv. # We link to them to save about 80Mb compared to cp/mv.
@ -25,132 +64,158 @@ for geodb in GeoLite2-ASN.mmdb GeoLite2-City.mmdb; do
fi fi
done done
#Check & prestage /etc/crowdsec # Check and prestage /etc/crowdsec
if [ ! -e "/etc/crowdsec/local_api_credentials.yaml" ] && [ ! -e "/etc/crowdsec/config.yaml" ]; then if [ ! -e "/etc/crowdsec/local_api_credentials.yaml" ] && [ ! -e "/etc/crowdsec/config.yaml" ]; then
mkdir -p /etc/crowdsec echo "Populating configuration directory..."
cp -r /staging/etc/* /etc/ # don't overwrite existing configuration files, which may come
# from bind-mount or even be read-only (configmaps)
if [ -e /staging/etc/crowdsec ]; then
mkdir -p /etc/crowdsec/
# if you change this, check that it still works
# under alpine and k8s, with and without tls
cp -an /staging/etc/crowdsec/* /etc/crowdsec/
fi
fi fi
# regenerate local agent credentials (ignore if agent is disabled) # regenerate local agent credentials (ignore if agent is disabled)
if [ "$DISABLE_AGENT" == "" ] ; then if isfalse "$DISABLE_AGENT"; then
if isfalse "$DISABLE_LOCAL_API"; then
echo "Regenerate local agent credentials" echo "Regenerate local agent credentials"
cscli -c "$CS_CONFIG_FILE" machines delete "${CUSTOM_HOSTNAME:-localhost}" cscli machines delete "$CUSTOM_HOSTNAME"
if [ "$LOCAL_API_URL" != "" ] ; then # shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" machines add "${CUSTOM_HOSTNAME:-localhost}" --auto --url "$LOCAL_API_URL" cscli machines add "$CUSTOM_HOSTNAME" --auto --url "$LOCAL_API_URL"
else
cscli -c "$CS_CONFIG_FILE" machines add "${CUSTOM_HOSTNAME:-localhost}" --auto
fi fi
if [ "$AGENT_USERNAME" != "" ] && [ "$AGENT_PASSWORD" != "" ] && [ "$LOCAL_API_URL" != "" ] ; then
echo "set up lapi credentials for agent" lapi_credentials_path=$(conf_get '.api.client.credentials_path')
CONFIG_PATH=$(yq eval '.api.client.credentials_path' "$CS_CONFIG_FILE" )
echo "url: $LOCAL_API_URL" > "$CONFIG_PATH" if istrue "$USE_TLS"; then
echo "login: $AGENT_USERNAME" >> "$CONFIG_PATH" install -m 0600 /dev/null "$lapi_credentials_path"
echo "password: $AGENT_PASSWORD" >> "$CONFIG_PATH" conf_set '
.url = strenv(LOCAL_API_URL) |
.ca_cert_path = strenv(CACERT_FILE) |
.key_path = strenv(KEY_FILE) |
.cert_path = strenv(CERT_FILE)
' "$lapi_credentials_path"
elif [ "$AGENT_USERNAME" != "" ]; then
install -m 0600 /dev/null "$lapi_credentials_path"
conf_set '
.url = strenv(LOCAL_API_URL) |
.login = strenv(AGENT_USERNAME) |
.password = strenv(AGENT_PASSWORD)
' "$lapi_credentials_path"
fi fi
fi fi
# Check if lapi needs to automatically register an agent if isfalse "$DISABLE_LOCAL_API"; then
echo "Check if lapi need to register automatically an agent" echo "Check if lapi needs to automatically register an agent"
if [ "$DISABLE_LOCAL_API" == "" ] && [ "$AGENT_USERNAME" != "" ] && [ "$AGENT_PASSWORD" != "" ] ; then
if [ "$LOCAL_API_URL" != "" ] ; then # pre-registration is not needed with TLS
cscli -c "$CS_CONFIG_FILE" machines add "$AGENT_USERNAME" --password "$AGENT_PASSWORD" --url "$LOCAL_API_URL" if isfalse "$USE_TLS" && [ "$AGENT_USERNAME" != "" ] && [ "$AGENT_PASSWORD" != "" ] ; then
else # shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" machines add "$AGENT_USERNAME" --password "$AGENT_PASSWORD" cscli machines add "$AGENT_USERNAME" --password "$AGENT_PASSWORD" --url "$LOCAL_API_URL"
fi
echo "Agent registered to lapi" echo "Agent registered to lapi"
fi
fi fi
# registration to online API for signal push # registration to online API for signal push
if [ "${DISABLE_ONLINE_API,,}" != "true" ] && [ "$CONFIG_FILE" == "" ] ; then if isfalse "$DISABLE_ONLINE_API" && [ "$CONFIG_FILE" == "/etc/crowdsec/config.yaml" ] ; then
CONFIG_EXIST=$(yq eval '.api.server.online_client | has("credentials_path")' "$CS_CONFIG_FILE") config_exists=$(conf_get '.api.server.online_client | has("credentials_path")')
if [ "$CONFIG_EXIST" != "true" ]; then if isfalse "$config_exists"; then
yq eval '.api.server.online_client = {"credentials_path": "/etc/crowdsec/online_api_credentials.yaml"}' "$CS_CONFIG_FILE" > /etc/crowdsec/config2.yaml conf_set '.api.server.online_client = {"credentials_path": "/etc/crowdsec/online_api_credentials.yaml"}'
mv /etc/crowdsec/config2.yaml "$CS_CONFIG_FILE" cscli capi register > /etc/crowdsec/online_api_credentials.yaml
cscli -c "$CS_CONFIG_FILE" capi register > /etc/crowdsec/online_api_credentials.yaml echo "Registration to online API done"
echo "registration to online API done"
fi fi
fi fi
## Enroll instance if enroll key is provided # Enroll instance if enroll key is provided
if [ "${DISABLE_ONLINE_API,,}" != "true" ] && [ "$ENROLL_KEY" != "" ] ; then if isfalse "$DISABLE_ONLINE_API" && [ "$ENROLL_KEY" != "" ]; then
enroll_args="" enroll_args=""
if [ "$ENROLL_INSTANCE_NAME" != "" ] ; then if [ "$ENROLL_INSTANCE_NAME" != "" ]; then
enroll_args="--name $ENROLL_INSTANCE_NAME" enroll_args="--name $ENROLL_INSTANCE_NAME"
fi fi
if [ "$ENROLL_TAGS" != "" ] ; then if [ "$ENROLL_TAGS" != "" ]; then
#shellcheck disable=SC2086 # shellcheck disable=SC2086
for tag in ${ENROLL_TAGS} for tag in ${ENROLL_TAGS}; do
do
enroll_args="$enroll_args --tags $tag" enroll_args="$enroll_args --tags $tag"
done done
fi fi
#shellcheck disable=SC2086 # shellcheck disable=SC2086
cscli console enroll $enroll_args "$ENROLL_KEY" cscli console enroll $enroll_args "$ENROLL_KEY"
fi fi
# crowdsec sqlite database permissions # crowdsec sqlite database permissions
if [ "$GID" != "" ]; then if [ "$GID" != "" ]; then
IS_SQLITE=$(yq eval '.db_config.type == "sqlite"' "$CS_CONFIG_FILE") if istrue "$(conf_get '.db_config.type == "sqlite"')"; then
DB_PATH=$(yq eval '.db_config.db_path' "$CS_CONFIG_FILE") chown ":$GID" "$(conf_get '.db_config.db_path')"
if [ "$IS_SQLITE" == "true" ]; then
chown ":$GID" "$DB_PATH"
echo "sqlite database permissions updated" echo "sqlite database permissions updated"
fi fi
fi fi
if [ "${USE_TLS,,}" == "true" ]; then if istrue "$USE_TLS"; then
yq -i eval ".api.server.tls.cert_file = \"$CERT_FILE\"" "$CS_CONFIG_FILE" agents_allowed_yaml=$(csv2yaml "$AGENTS_ALLOWED_OU") \
yq -i eval ".api.server.tls.key_file = \"$KEY_FILE\"" "$CS_CONFIG_FILE" bouncers_allowed_yaml=$(csv2yaml "$BOUNCERS_ALLOWED_OU") \
yq -i eval '... comments=""' "$CS_CONFIG_FILE" conf_set '
.api.server.tls.ca_cert_path = strenv(CACERT_FILE) |
.api.server.tls.cert_file = strenv(CERT_FILE) |
.api.server.tls.key_file = strenv(KEY_FILE) |
.api.server.tls.bouncers_allowed_ou = env(bouncers_allowed_yaml) |
.api.server.tls.agents_allowed_ou = env(agents_allowed_yaml) |
... comments=""
'
fi fi
if [ "$PLUGIN_DIR" != "/usr/local/lib/crowdsec/plugins/" ]; then conf_set ".config_paths.plugin_dir = strenv(PLUGIN_DIR)"
yq -i eval ".config_paths.plugin_dir = \"$PLUGIN_DIR\"" "$CS_CONFIG_FILE"
fi
## Install collections, parsers, scenarios & postoverflows ## Install collections, parsers, scenarios & postoverflows
cscli -c "$CS_CONFIG_FILE" hub update cscli hub update
cscli -c "$CS_CONFIG_FILE" collections upgrade crowdsecurity/linux || true cscli collections upgrade crowdsecurity/linux || true
cscli -c "$CS_CONFIG_FILE" parsers upgrade crowdsecurity/whitelists || true cscli parsers upgrade crowdsecurity/whitelists || true
cscli -c "$CS_CONFIG_FILE" parsers install crowdsecurity/docker-logs || true cscli parsers install crowdsecurity/docker-logs || true
if [ "$COLLECTIONS" != "" ]; then if [ "$COLLECTIONS" != "" ]; then
#shellcheck disable=SC2086 # shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" collections install $COLLECTIONS cscli collections install $COLLECTIONS
fi fi
if [ "$PARSERS" != "" ]; then if [ "$PARSERS" != "" ]; then
#shellcheck disable=SC2086 # shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" parsers install $PARSERS cscli parsers install $PARSERS
fi fi
if [ "$SCENARIOS" != "" ]; then if [ "$SCENARIOS" != "" ]; then
#shellcheck disable=SC2086 # shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" scenarios install $SCENARIOS cscli scenarios install $SCENARIOS
fi fi
if [ "$POSTOVERFLOWS" != "" ]; then if [ "$POSTOVERFLOWS" != "" ]; then
#shellcheck disable=SC2086 # shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" postoverflows install $POSTOVERFLOWS cscli postoverflows install $POSTOVERFLOWS
fi fi
## Remove collections, parsers, scenarios & postoverflows ## Remove collections, parsers, scenarios & postoverflows
if [ "$DISABLE_COLLECTIONS" != "" ]; then if [ "$DISABLE_COLLECTIONS" != "" ]; then
#shellcheck disable=SC2086 # shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" collections remove $DISABLE_COLLECTIONS cscli collections remove $DISABLE_COLLECTIONS
fi
if [ "$DISABLE_PARSERS" != "" ]; then
#shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" parsers remove $DISABLE_PARSERS
fi
if [ "$DISABLE_SCENARIOS" != "" ]; then
#shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" scenarios remove $DISABLE_SCENARIOS
fi
if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
#shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" postoverflows remove $DISABLE_POSTOVERFLOWS
fi fi
function register_bouncer { if [ "$DISABLE_PARSERS" != "" ]; then
if ! cscli -c "$CS_CONFIG_FILE" bouncers list -o json | sed '/^ *"name"/!d;s/^ *"name": "\(.*\)",/\1/' | grep -q "^${NAME}$"; then # shellcheck disable=SC2086
if cscli -c "$CS_CONFIG_FILE" bouncers add "${NAME}" -k "${KEY}" > /dev/null; then cscli parsers remove $DISABLE_PARSERS
fi
if [ "$DISABLE_SCENARIOS" != "" ]; then
# shellcheck disable=SC2086
cscli scenarios remove $DISABLE_SCENARIOS
fi
if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
# shellcheck disable=SC2086
cscli postoverflows remove $DISABLE_POSTOVERFLOWS
fi
register_bouncer() {
if ! cscli bouncers list -o json | sed '/^ *"name"/!d;s/^ *"name": "\(.*\)",/\1/' | grep -q "^${NAME}$"; then
if cscli bouncers add "${NAME}" -k "${KEY}" > /dev/null; then
echo "Registered bouncer for ${NAME}" echo "Registered bouncer for ${NAME}"
else else
echo "Failed to register bouncer for ${NAME}" echo "Failed to register bouncer for ${NAME}"
@ -182,6 +247,7 @@ ARGS=""
if [ "$CONFIG_FILE" != "" ]; then if [ "$CONFIG_FILE" != "" ]; then
ARGS="-c $CONFIG_FILE" ARGS="-c $CONFIG_FILE"
fi fi
if [ "$DSN" != "" ]; then if [ "$DSN" != "" ]; then
ARGS="$ARGS -dsn ${DSN}" ARGS="$ARGS -dsn ${DSN}"
fi fi
@ -189,24 +255,32 @@ fi
if [ "$TYPE" != "" ]; then if [ "$TYPE" != "" ]; then
ARGS="$ARGS -type $TYPE" ARGS="$ARGS -type $TYPE"
fi fi
if [ "${TEST_MODE,,}" == "true" ]; then
if istrue "$TEST_MODE"; then
ARGS="$ARGS -t" ARGS="$ARGS -t"
fi fi
if [ "${DISABLE_AGENT,,}" == "true" ]; then
if istrue "$DISABLE_AGENT"; then
ARGS="$ARGS -no-cs" ARGS="$ARGS -no-cs"
fi fi
if [ "${DISABLE_LOCAL_API,,}" == "true" ]; then
if istrue "$DISABLE_LOCAL_API"; then
ARGS="$ARGS -no-api" ARGS="$ARGS -no-api"
fi fi
if [ "${LEVEL_TRACE,,}" == "true" ]; then
if istrue "$LEVEL_TRACE"; then
ARGS="$ARGS -trace" ARGS="$ARGS -trace"
fi fi
if [ "${LEVEL_DEBUG,,}" == "true" ]; then
if istrue "$LEVEL_DEBUG"; then
ARGS="$ARGS -debug" ARGS="$ARGS -debug"
fi fi
if [ "${LEVEL_INFO,,}" == "true" ]; then
if istrue "$LEVEL_INFO"; then
ARGS="$ARGS -info" ARGS="$ARGS -info"
fi fi
#shellcheck disable=SC2086 conf_set '.prometheus.listen_port=env(METRICS_PORT)'
# shellcheck disable=SC2086
exec crowdsec $ARGS exec crowdsec $ARGS

4
pkg/apiserver/apiserver.go
View file

@ -322,11 +322,11 @@ func (s *APIServer) Run(apiReady chan bool) error {
log.Infof("CrowdSec Local API listening on %s", s.URL) log.Infof("CrowdSec Local API listening on %s", s.URL)
if s.TLS != nil && s.TLS.CertFilePath != "" && s.TLS.KeyFilePath != "" { if s.TLS != nil && s.TLS.CertFilePath != "" && s.TLS.KeyFilePath != "" {
if err := s.httpServer.ListenAndServeTLS(s.TLS.CertFilePath, s.TLS.KeyFilePath); err != nil { if err := s.httpServer.ListenAndServeTLS(s.TLS.CertFilePath, s.TLS.KeyFilePath); err != nil {
log.Fatal(err) log.Fatalf("while serving local API: %v", err)
} }
} else { } else {
if err := s.httpServer.ListenAndServe(); err != http.ErrServerClosed { if err := s.httpServer.ListenAndServe(); err != http.ErrServerClosed {
log.Fatal(err) log.Fatalf("while serving local API: %v", err)
} }
} }
}() }()