beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Docker refactoring, tls setup (#1869)

Browse source
This commit is contained in:
mmetc 2022-11-28 10:35:12 +01:00 committed by GitHub
parent c5079ac15e
commit fde9640364
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 377 additions and 156 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
Dockerfile
View file

@ -9,7 +9,7 @@ COPY . .
# wizard.sh requires GNU coreutils
RUN apk add --no-cache git gcc libc-dev make bash gettext binutils-gold coreutils && \
SYSTEM="docker" make release && \
SYSTEM="docker" make clean release && \
cd crowdsec-v* && \
./wizard.sh --docker-mode && \
cd - && \
@ -23,15 +23,72 @@ RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/co
mkdir -p /staging/etc/crowdsec && \
mkdir -p /staging/var/lib/crowdsec && \
mkdir -p /var/lib/crowdsec/data
COPY --from=build /etc/crowdsec /staging/etc/crowdsec
COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
ENV CONFIG_FILE=/etc/crowdsec/config.yaml
ENV LOCAL_API_URL=http://0.0.0.0:8080/
ENV CUSTOM_HOSTNAME=localhost
ENV PLUGIN_DIR=/usr/local/lib/crowdsec/plugins/
ENV DISABLE_AGENT=false
ENV DISABLE_LOCAL_API=false
ENV DISABLE_ONLINE_API=false
ENV DSN=
ENV TYPE=
ENV TEST_MODE=false
# register to app.crowdsec.net
ENV ENROLL_INSTANCE_NAME=
ENV ENROLL_KEY=
ENV ENROLL_TAGS=
# log verbosity
ENV LEVEL_TRACE=false
ENV LEVEL_DEBUG=false
ENV LEVEL_INFO=true
# TLS setup ----------------------------------- #
ENV AGENT_USERNAME=
ENV AGENT_PASSWORD=
# TLS setup ----------------------------------- #
ENV USE_TLS=false
ENV CA_CERT_PATH=
ENV CERT_FILE=/etc/ssl/cert.pem
ENV KEY_FILE=/etc/ssl/key.pem
# comma-separated list of allowed OU values for TLS bouncer certificates
ENV BOUNCERS_ALLOWED_OU=bouncer-ou
# comma-separated list of allowed OU values for TLS agent certificates
ENV AGENTS_ALLOWED_OU=agent-ou
# Install the following hub items --------------#
ENV COLLECTIONS=
ENV PARSERS=
ENV SCENARIOS=
ENV POSTOVERFLOWS=
# Uninstall the following hub items ------------#
ENV DISABLE_COLLECTIONS=
ENV DISABLE_PARSERS=
ENV DISABLE_SCENARIOS=
ENV DISABLE_POSTOVERFLOWS=
ENV METRICS_PORT=6060
ENTRYPOINT /bin/bash docker_start.sh
FROM build-slim as build-plugins
# Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
# The files are here for reference, as users will need to mount a new version to be actually able to use notifications
COPY --from=build /go/src/crowdsec/plugins/notifications/email/email.yaml /staging/etc/crowdsec/notifications/email.yaml

56
Dockerfile.debian
View file

@ -1,3 +1,4 @@
# vim: set ft=dockerfile:
ARG BUILD_ENV=full
ARG GOVERSION=1.19
@ -41,6 +42,61 @@ COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
RUN yq eval -i ".plugin_config.group = \"nogroup\"" /staging/etc/crowdsec/config.yaml
ENV CONFIG_FILE=/etc/crowdsec/config.yaml
ENV LOCAL_API_URL=http://0.0.0.0:8080/
ENV CUSTOM_HOSTNAME=localhost
ENV PLUGIN_DIR=/usr/local/lib/crowdsec/plugins/
ENV DISABLE_AGENT=false
ENV DISABLE_LOCAL_API=false
ENV DISABLE_ONLINE_API=false
ENV DSN=
ENV TYPE=
ENV TEST_MODE=false
# register to app.crowdsec.net
ENV ENROLL_INSTANCE_NAME=
ENV ENROLL_KEY=
ENV ENROLL_TAGS=
# log verbosity
ENV LEVEL_TRACE=false
ENV LEVEL_DEBUG=false
ENV LEVEL_INFO=true
# TLS setup ----------------------------------- #
ENV AGENT_USERNAME=
ENV AGENT_PASSWORD=
# TLS setup ----------------------------------- #
ENV USE_TLS=false
ENV CA_CERT_PATH=
ENV CERT_FILE=/etc/ssl/cert.pem
ENV KEY_FILE=/etc/ssl/key.pem
# comma-separated list of allowed OU values for TLS bouncer certificates
ENV BOUNCERS_ALLOWED_OU=bouncer-ou
# comma-separated list of allowed OU values for TLS agent certificates
ENV AGENTS_ALLOWED_OU=agent-ou
# Install the following hub items --------------#
ENV COLLECTIONS=
ENV PARSERS=
ENV SCENARIOS=
ENV POSTOVERFLOWS=
# Uninstall the following hub items ------------#
ENV DISABLE_COLLECTIONS=
ENV DISABLE_PARSERS=
ENV DISABLE_SCENARIOS=
ENV DISABLE_POSTOVERFLOWS=
ENV METRICS_PORT=6060
ENTRYPOINT /bin/bash docker_start.sh
FROM build-slim as build-plugins

144
docker/README.md
View file

@ -10,23 +10,24 @@ Crowdsec - An open-source, lightweight agent to detect and respond to bad behavi
# How to use this image
## Docker image versions
## Image flavors
All the following versions are available on Docker Hub for 386, amd64, arm/v6, arm/v7, arm64.
All the following images are available on Docker Hub for the architectures
386, amd64, arm/v6, arm/v7, arm64.
### Alpine
- crowdsecurity/crowdsec:{version}
- `crowdsecurity/crowdsec:{version}`
Recommended for production usage. Also available on GitHub (ghrc.io).
- crowdsecurity/crowdsec:latest
- `crowdsecurity/crowdsec:latest`
For development and testing.
since v1.4.2:
- crowdsecurity/crowdsec:slim
- `crowdsecurity/crowdsec:slim`
Reduced size by 60%, does not include notifier plugins nor the GeoIP database.
If you need these details on decisions, running `cscli hub upgrade` inside the
@ -35,14 +36,14 @@ container downloads the GeoIP database at runtime.
### Debian (since v1.3.3)
- crowdsecurity/crowdsec:{version}-debian
- crowdsecurity/crowdsec:latest-debian
- `crowdsecurity/crowdsec:{version}-debian`
- `crowdsecurity/crowdsec:latest-debian`
The debian version includes support for systemd and journalctl.
### Custom
You can build your own images with Dockerfile and Dockerfile-debian.
You can build your images with Dockerfile and Dockerfile-debian.
For example, if you want a Debian version without plugin notifiers:
@ -56,8 +57,8 @@ supported values for BUILD_ENV are: full, with-geoip, with-plugins, slim.
## Required configuration
### Journalctl (only for debian image)
To use journalctl (only with the debian image) as a log stream, eventually from the `DSN` environment variable, it's important to mount the journal log from the host to the container itself.
This can be done by adding the following volume mount to your docker command:
To use journalctl as a log stream, with or without the `DSN` environment variable, it's important to mount the journal log from the host to the container itself.
This can be done by adding the following volume mount to the docker command:
```
-v /var/log/journal:/run/log/journal
@ -66,14 +67,14 @@ This can be done by adding the following volume mount to your docker command:
### Logs ingestion and processing
Collections are a good place to start: https://docs.crowdsec.net/docs/collections/intro
Find collections|scenarios|parsers|postoverflows in the hub: https://hub.crowdsec.net
Find collections, scenarios, parsers and postoverflows in the hub: https://hub.crowdsec.net
* Specify collections|scenarios|parsers/postoverflows to install via the environment variables (by default [`crowdsecurity/linux`](https://hub.crowdsec.net/author/crowdsecurity/collections/linux) is installed)
* Specify collections | scenarios | parsers | postoverflows to install via the environment variables (by default [`crowdsecurity/linux`](https://hub.crowdsec.net/author/crowdsecurity/collections/linux) is installed)
* Mount volumes to specify your log files that should be ingested by crowdsec
### Acquisition
`/etc/crowdsec/acquis.yaml` maps logs to provided parsers. Find out more here: https://docs.crowdsec.net/docs/concepts/#acquisition
`/etc/crowdsec/acquis.yaml` maps logs to the provided parsers. Find out more here: https://docs.crowdsec.net/docs/concepts/#acquisition
acquis.yaml example:
```shell
@ -88,14 +89,15 @@ labels:
type: apache2
```
`labels.type`: use `syslog` if logs origin is `syslog`, checkout collection's documentation for the relevant type otherwise.
`labels.type`: use `syslog` if the logs come from syslog, otherwise check the collection's documentation for the relevant type.
## Recommended configuration
### Volumes
We strongly suggest to mount **named volumes** for Crowdsec configuration and database to avoid credentials and decisions loss in case of container's destruction and recreation, version update, etc.
We strongly suggest mounting **named volumes** for Crowdsec configuration and database to avoid credentials and decisions loss in case of container destruction and recreation, version update, etc.
* Credentials and configuration: `/etc/crowdsec`
* Database when using default SQLite: `/var/lib/crowdsec/data`
* Database when using SQLite (default): `/var/lib/crowdsec/data`
## Start a Crowdsec instance
@ -114,15 +116,26 @@ docker run -d \
## ... or docker-compose
Check this full stack example using docker-compose: https://github.com/crowdsecurity/example-docker-compose
Check this full-stack example using docker-compose: https://github.com/crowdsecurity/example-docker-compose
# How to extend this image
## Full configuration
The container is built with a specific docker [configuration](https://github.com/crowdsecurity/crowdsec/blob/master/docker/config.yaml). If you need to change it, bind `/etc/crowdsec/config.yaml` to your local configuration file
The container is built with a specific docker
[configuration](https://github.com/crowdsecurity/crowdsec/blob/master/docker/config.yaml).
If you need to change it and the docker variables (see below) are not enough,
you can bind `/etc/crowdsec/config.yaml` to your a configuration file.
## Notifications
If you wish to use the [notification system](https://docs.crowdsec.net/docs/notification_plugins/intro), you will need to mount at least a custom `profiles.yaml` and a notification configuration to `/etc/crowdsec/notifications`
# Deployment use cases
Crowdsec is composed of an `agent` that parses logs and creates `alerts` that `local API` or `LAPI` transform into decisions. Both can run in the same process but also on separated containers as it makes sense in complex configurations to have agents on the same machines as the protected component and a LAPI that gather all signals from agents and communicate with the `central api`.
Crowdsec is composed of an `agent` that parses logs and creates `alerts`, and a
`local API (LAPI)` that transforms these alerts into decisions. Both functions
are provided by the same executables, so the agent and the LAPI can run in the
same or separate containers. In complex configurations, it makes sense to have
agents on each machine that runs the protected applications, and a LAPI that
gathers all signals from agents and communicates with the `central API`.
## Register a new agent with LAPI
```shell
@ -130,15 +143,19 @@ docker exec -it crowdsec_lapi_container_name cscli machines add agent_user_name
```
## Run an agent connected to LAPI
Add the following environment variables to your docker run command:
Add the following environment variables to the docker run command:
* `DISABLE_LOCAL_API=true`
* `AGENT_USERNAME="agent_user_name"` - agent_user_name previously registered with LAPI
* `AGENT_PASSWORD="agent_password"` - agent_password previously registered with LAPI
* `LOCAL_API_URL="http://LAPI_host:LAPI_port"`
# Next steps
## Bouncers
Crowdsec being a detection component, remediation is implemented using `bouncers`. Each bouncer protects a specific component. Find out more:
Crowdsec being a detection component, the remediation is implemented using `bouncers`. Each bouncer protects a specific component. Find out more:
https://hub.crowdsec.net/browse/#bouncers
@ -146,13 +163,13 @@ https://docs.crowdsec.net/docs/user_guides/bouncers_configuration/
### Automatic Bouncer Registration
You can automatically register bouncers with the crowdsec container on startup using environment variables or Docker secrets. You cannot use this process to update an existing bouncer without first deleting it.
You can automatically register bouncers with the crowdsec container at startup, using environment variables or Docker secrets. You cannot use this process to update an existing bouncer without first deleting it.
To use environment variables, they should be in the format `BOUNCER_KEY_<name>=<key>`. e.g. `BOUNCER_KEY_nginx=mysecretkey12345`.
To use Docker secrets, the secret should be named `bouncer_key_<name>` with a content of `<key>`. e.g. `bouncer_key_nginx` with content `mysecretkey12345`.
A bouncer key can be any string but we recommend an alphanumeric value to keep consistent with crowdsec-generated keys and avoid problems with escaping special characters.
A bouncer key can be any string but we recommend an alphanumeric value for consistency with crowdsec-generated keys and avoid problems with escaping special characters.
## Console
We provide a web-based interface to get more from Crowdsec: https://docs.crowdsec.net/docs/console
@ -160,42 +177,59 @@ We provide a web-based interface to get more from Crowdsec: https://docs.crowdse
Subscribe here: https://app.crowdsec.net
# Caveats
Using binds rather than named volumes ([more explanation here](https://docs.docker.com/storage/volumes/)) results in more complexity as you'll have to bind relevant files one by one whereas with named volumes you can mount full configuration and data folders. On the other hand, named volumes are less straightforward to navigate.
Using binds rather than named volumes ([complete explanation here](https://docs.docker.com/storage/volumes/)) results in more complexity as you'll have to bind the relevant files one by one whereas with named volumes you can mount full configuration and data folders. On the other hand, named volumes are less straightforward to navigate.
# Reference
## Environment Variables
* `COLLECTIONS` - Collections to install from the [hub](https://hub.crowdsec.net/browse/#collections), separated by space : `-e COLLECTIONS="crowdsecurity/linux crowdsecurity/apache2"`
* `SCENARIOS` - Scenarios to install from the [hub](https://hub.crowdsec.net/browse/#configurations), separated by space : `-e SCENARIOS="crowdsecurity/http-bad-user-agent crowdsecurity/http-xss-probing"`
* `PARSERS` - Parsers to install from the [hub](https://hub.crowdsec.net/browse/#configurations), separated by space : `-e PARSERS="crowdsecurity/http-logs crowdsecurity/modsecurity"`
* `POSTOVERFLOWS` - Postoverflows to install from the [hub](https://hub.crowdsec.net/browse/#configurations), separated by space : `-e POSTOVERFLOWS="crowdsecurity/cdn-whitelist"`
* `CONFIG_FILE` - Configuration file (default: `/etc/crowdsec/config.yaml`) : `-e CONFIG_FILE="<config_path>"`
* `DSN` - Process a single source in time-machine : `-e DSN="file:///var/log/toto.log"` or `-e DSN="cloudwatch:///your/group/path:stream_name?profile=dev&backlog=16h"` or `-e DSN="journalctl://filters=_SYSTEMD_UNIT=ssh.service"`
* `TYPE` - [`Labels.type`](https://docs.crowdsec.net/Crowdsec/v1/references/acquisition/) for file in time-machine : `-e TYPE="<type>"`
* `TEST_MODE` - Only test configs (default: `false`) : `-e TEST_MODE="<true|false>"`
* `TZ` - Set the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) to ensure logs have a local timestamp.
* `DISABLE_AGENT` - Only test configs (default: `false`) : `-e DISABLE_AGENT="<true|false>"`
* `DISABLE_LOCAL_API` - Disable local API (default: `false`) : `-e DISABLE_LOCAL_API="<true|false>"`
* `AGENT_USERNAME` - Agent username (to register if is LAPI or to use if it's an agent) : `-e AGENT_USERNAME="machine_id"`
* `AGENT_PASSWORD` - Agent password (to register if is LAPI or to use if it's an agent) : `-e AGENT_PASSWORD="machine_password"`
* `LOCAL_API_URL` - To specify when an agent needs to connect to a LAPI crowdsec (To use only when `DISABLE_LOCAL_API` is set to `true`) : `-e LOCAL_API_URL="http://lapi-address:8080"`
* `DISABLE_ONLINE_API` - Disable Online API registration for signal sharing (default: `false`) : `-e DISABLE_ONLINE_API="<true|false>"`
* `LEVEL_TRACE` - Trace-level (VERY verbose) on stdout (default: `false`) : `-e LEVEL_TRACE="<true|false>"`
* `LEVEL_DEBUG` - Debug-level on stdout (default: `false`) : `-e LEVEL_DEBUG="<true|false>"`
* `LEVEL_INFO` - Info-level on stdout (default: `false`) : `-e LEVEL_INFO="<true|false>"`
* `USE_TLS` - Enable TLS on the API Server (default: `false`) : `-e USE_TLS="<true|false>"`
* `CERT_FILE` - TLS Certificate file (default: `/etc/ssl/cert.pem`) : `-e CERT_FILE="<file_path>"`
* `KEY_FILE` - TLS Key file (default: `/etc/ssl/key.pem`) : `-e KEY_FILE="<file_path>"`
* `CUSTOM_HOSTNAME` - Custom hostname for local api (default: `localhost`) : `-e CUSTOM_HOSTNAME="<hostname>"`
* `DISABLE_COLLECTIONS` - Collections to remove from the [hub](https://hub.crowdsec.net/browse/#collections), separated by space : `-e DISABLE_COLLECTIONS="crowdsecurity/linux crowdsecurity/nginx"`
* `DISABLE_PARSERS` - Parsers to remove from the [hub](https://hub.crowdsec.net/browse/#configurations), separated by space : `-e DISABLE_PARSERS="crowdsecurity/apache2-logs crowdsecurity/nginx-logs"`
* `DISABLE_SCENARIOS` - Scenarios to remove from the [hub](https://hub.crowdsec.net/browse/#configurations), separated by space : `-e DISABLE_SCENARIOS="crowdsecurity/http-bad-user-agent crowdsecurity/http-xss-probing"`
* `DISABLE_POSTOVERFLOWS` - Postoverflows to remove from the [hub](https://hub.crowdsec.net/browse/#configurations), separated by space : `-e DISABLE_POSTOVERFLOWS="crowdsecurity/cdn-whitelist crowdsecurity/seo-bots-whitelist"`
* `PLUGIN_DIR` - Directory for plugins (default: `/usr/local/lib/crowdsec/plugins/`) : `-e PLUGIN_DIR="<path>"`
* `BOUNCER_KEY_<name>` - Register a bouncer with the name `<name>` and a key equal to the value of the environment variable.
* `ENROLL_KEY` - Enroll key retrieved from [the console](https://app.crowdsec.net/) to enroll the instance.
* `ENROLL_INSTANCE_NAME` - To set an instance name and see it on [the console](https://app.crowdsec.net/).
* `ENROLL_TAGS` - To set tags when enrolling an instance and use them for search and filtering on [the console](https://app.crowdsec.net/)
| Variable | Default | Description |
| ----------------------- | ------------------------- | ----------- |
| `CONFIG_FILE` | `/etc/crowdsec/config.yaml` | Configuration file location |
| `DSN` | | Process a single source in time-machine: `-e DSN="file:///var/log/toto.log"` or `-e DSN="cloudwatch:///your/group/path:stream_name?profile=dev&backlog=16h"` or `-e DSN="journalctl://filters=_SYSTEMD_UNIT=ssh.service"` |
| `TYPE` | | [`Labels.type`](https://docs.crowdsec.net/Crowdsec/v1/references/acquisition/) for file in time-machine: `-e TYPE="<type>"` |
| `TEST_MODE` | false | Don't run the service, only test the configuration: `-e TEST_MODE=true` |
| `TZ` | | Set the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) to ensure the logs have a local timestamp. |
| `LOCAL_API_URL` | `http://0.0.0.0:8080` | The LAPI URL, you need to change this when `DISABLE_LOCAL_API` is true: `-e LOCAL_API_URL="http://lapi-address:8080"` |
| `DISABLE_AGENT` | false | Disable the agent, run a LAPI-only container |
| `DISABLE_LOCAL_API` | false | Disable LAPI, run an agent-only container |
| `DISABLE_ONLINE_API` | false | Disable online API registration for signal sharing |
| `CUSTOM_HOSTNAME` | localhost | Custom hostname for LAPI registration (with agent and LAPI on the same container) |
| `PLUGIN_DIR` | `/usr/local/lib/crowdsec/plugins/` | Directory for plugins: `-e PLUGIN_DIR="<path>"` |
| `BOUNCER_KEY_<name>` | | Register a bouncer with the name `<name>` and a key equal to the value of the environment variable. |
| `METRICS_PORT` | 6060 | Port to expose Prometheus metrics |
| | | |
| __Console__ | | |
| `ENROLL_KEY` | | Enroll key retrieved from [the console](https://app.crowdsec.net/) to enroll the instance. |
| `ENROLL_INSTANCE_NAME` | | To set an instance name and see it on [the console](https://app.crowdsec.net/) |
| `ENROLL_TAGS` | | Tags of the enrolled instance, for search and filter |
| | | |
| __Password Auth__ | | |
| `AGENT_USERNAME` | | Agent username (to register if is LAPI or to use if it's an agent): `-e AGENT_USERNAME="machine_id"` |
| `AGENT_PASSWORD` | | Agent password (to register if is LAPI or to use if it's an agent): `-e AGENT_PASSWORD="machine_password"` |
| | | |
| __TLS Auth/encryption | | |
| `USE_TLS` | false | Enable TLS on the LAPI |
| `CERT_FILE` | /etc/ssl/cert.pem | TLS Certificate path |
| `KEY_FILE` | /etc/ssl/key.pem | TLS Key path |
| `CACERT_FILE` | | CA certificate |
| `AGENTS_ALLOWED_OU` | agent-ou | OU values allowed for agents, separated by comma |
| `BOUNCERS_ALLOWED_OU` | bouncer-ou | OU values allowed for bouncers, separated by comma |
| | | |
| __Hub management__ | | |
| `COLLECTIONS` | | Collections to install, separated by space: `-e COLLECTIONS="crowdsecurity/linux crowdsecurity/apache2"` |
| `SCENARIOS` | | Scenarios to install, separated by space |
| `PARSERS` | | Parsers to install, separated by space |
| `POSTOVERFLOWS` | | Postoverflows to install, separated by space |
| `DISABLE_COLLECTIONS` | | Collections to remove, separated by space: `-e DISABLE_COLLECTIONS="crowdsecurity/linux crowdsecurity/nginx"` |
| `DISABLE_PARSERS` | | Parsers to remove, separated by space |
| `DISABLE_SCENARIOS` | | Scenarios to remove, separated by space |
| `DISABLE_POSTOVERFLOWS` | | Postoverflows to remove, separated by space |
| | | |
| __Log verbosity__ | | |
| `LEVEL_INFO` | false | Force INFO level for the container log |
| `LEVEL_DEBUG` | false | Force DEBUG level for the container log |
| `LEVEL_TRACE` | false | Force TRACE level (VERY verbose) for the container log |
## Volumes

270
docker/docker_start.sh
View file

@ -1,21 +1,60 @@
#!/bin/bash
# Set the crowdsec config file
CS_CONFIG_FILE="/etc/crowdsec/config.yaml"
if [ "$CONFIG_FILE" != "" ]; then
CS_CONFIG_FILE="$CONFIG_FILE"
fi
# shellcheck disable=SC2292 # allow [ test ] syntax
# shellcheck disable=SC2310 # allow "if function..." syntax with -e
# TLS defaults
CERT_FILE="${CERT_FILE:-/etc/ssl/cert.pem}"
KEY_FILE="${KEY_FILE:-/etc/ssl/key.pem}"
set -e
shopt -s inherit_errexit
# Plugins directory default
PLUGIN_DIR="${PLUGIN_DIR:-/usr/local/lib/crowdsec/plugins/}"
#- HELPER FUNCTIONS ----------------#
# Check & prestage databases
istrue() {
case "$(echo "$1" | tr '[:upper:]' '[:lower:]')" in
true) return 0 ;;
*) return 1 ;;
esac
}
isfalse() {
if istrue "$1"; then
return 1
else
return 0
fi
}
# generate a yaml list from a comma-separated string of values
csv2yaml() {
[ -z "$1" ] && return
echo "$1" | sed 's/,/\n- /g;s/^/- /g'
}
# wrap cscli with the correct config file location
cscli() {
command cscli -c "$CONFIG_FILE" "$@"
}
conf_get() {
if [ $# -ge 2 ]; then
yq e "$1" "$2"
else
yq e "$1" "$CONFIG_FILE"
fi
}
conf_set() {
if [ $# -ge 2 ]; then
yq e "$1" -i "$2"
else
yq e "$1" -i "$CONFIG_FILE"
fi
}
#-----------------------------------#
# Check and prestage databases
for geodb in GeoLite2-ASN.mmdb GeoLite2-City.mmdb; do
# We keep the pre-populated geoib databases in /staging instead of /var,
# We keep the pre-populated geoip databases in /staging instead of /var,
# because if the data directory is bind-mounted from the host, it will be
# empty and the files will be out of reach, requiring a runtime download.
# We link to them to save about 80Mb compared to cp/mv.
@ -25,132 +64,158 @@ for geodb in GeoLite2-ASN.mmdb GeoLite2-City.mmdb; do
fi
done
#Check & prestage /etc/crowdsec
# Check and prestage /etc/crowdsec
if [ ! -e "/etc/crowdsec/local_api_credentials.yaml" ] && [ ! -e "/etc/crowdsec/config.yaml" ]; then
mkdir -p /etc/crowdsec
cp -r /staging/etc/* /etc/
echo "Populating configuration directory..."
# don't overwrite existing configuration files, which may come
# from bind-mount or even be read-only (configmaps)
if [ -e /staging/etc/crowdsec ]; then
mkdir -p /etc/crowdsec/
# if you change this, check that it still works
# under alpine and k8s, with and without tls
cp -an /staging/etc/crowdsec/* /etc/crowdsec/
fi
fi
# regenerate local agent credentials (ignore if agent is disabled)
if [ "$DISABLE_AGENT" == "" ] ; then
echo "Regenerate local agent credentials"
cscli -c "$CS_CONFIG_FILE" machines delete "${CUSTOM_HOSTNAME:-localhost}"
if [ "$LOCAL_API_URL" != "" ] ; then
cscli -c "$CS_CONFIG_FILE" machines add "${CUSTOM_HOSTNAME:-localhost}" --auto --url "$LOCAL_API_URL"
else
cscli -c "$CS_CONFIG_FILE" machines add "${CUSTOM_HOSTNAME:-localhost}" --auto
if isfalse "$DISABLE_AGENT"; then
if isfalse "$DISABLE_LOCAL_API"; then
echo "Regenerate local agent credentials"
cscli machines delete "$CUSTOM_HOSTNAME"
# shellcheck disable=SC2086
cscli machines add "$CUSTOM_HOSTNAME" --auto --url "$LOCAL_API_URL"
fi
if [ "$AGENT_USERNAME" != "" ] && [ "$AGENT_PASSWORD" != "" ] && [ "$LOCAL_API_URL" != "" ] ; then
echo "set up lapi credentials for agent"
CONFIG_PATH=$(yq eval '.api.client.credentials_path' "$CS_CONFIG_FILE" )
echo "url: $LOCAL_API_URL" > "$CONFIG_PATH"
echo "login: $AGENT_USERNAME" >> "$CONFIG_PATH"
echo "password: $AGENT_PASSWORD" >> "$CONFIG_PATH"
lapi_credentials_path=$(conf_get '.api.client.credentials_path')
if istrue "$USE_TLS"; then
install -m 0600 /dev/null "$lapi_credentials_path"
conf_set '
.url = strenv(LOCAL_API_URL) |
.ca_cert_path = strenv(CACERT_FILE) |
.key_path = strenv(KEY_FILE) |
.cert_path = strenv(CERT_FILE)
' "$lapi_credentials_path"
elif [ "$AGENT_USERNAME" != "" ]; then
install -m 0600 /dev/null "$lapi_credentials_path"
conf_set '
.url = strenv(LOCAL_API_URL) |
.login = strenv(AGENT_USERNAME) |
.password = strenv(AGENT_PASSWORD)
' "$lapi_credentials_path"
fi
fi
# Check if lapi needs to automatically register an agent
echo "Check if lapi need to register automatically an agent"
if [ "$DISABLE_LOCAL_API" == "" ] && [ "$AGENT_USERNAME" != "" ] && [ "$AGENT_PASSWORD" != "" ] ; then
if [ "$LOCAL_API_URL" != "" ] ; then
cscli -c "$CS_CONFIG_FILE" machines add "$AGENT_USERNAME" --password "$AGENT_PASSWORD" --url "$LOCAL_API_URL"
else
cscli -c "$CS_CONFIG_FILE" machines add "$AGENT_USERNAME" --password "$AGENT_PASSWORD"
if isfalse "$DISABLE_LOCAL_API"; then
echo "Check if lapi needs to automatically register an agent"
# pre-registration is not needed with TLS
if isfalse "$USE_TLS" && [ "$AGENT_USERNAME" != "" ] && [ "$AGENT_PASSWORD" != "" ] ; then
# shellcheck disable=SC2086
cscli machines add "$AGENT_USERNAME" --password "$AGENT_PASSWORD" --url "$LOCAL_API_URL"
echo "Agent registered to lapi"
fi
echo "Agent registered to lapi"
fi
# registration to online API for signal push
if [ "${DISABLE_ONLINE_API,,}" != "true" ] && [ "$CONFIG_FILE" == "" ] ; then
CONFIG_EXIST=$(yq eval '.api.server.online_client | has("credentials_path")' "$CS_CONFIG_FILE")
if [ "$CONFIG_EXIST" != "true" ]; then
yq eval '.api.server.online_client = {"credentials_path": "/etc/crowdsec/online_api_credentials.yaml"}' "$CS_CONFIG_FILE" > /etc/crowdsec/config2.yaml
mv /etc/crowdsec/config2.yaml "$CS_CONFIG_FILE"
cscli -c "$CS_CONFIG_FILE" capi register > /etc/crowdsec/online_api_credentials.yaml
echo "registration to online API done"
if isfalse "$DISABLE_ONLINE_API" && [ "$CONFIG_FILE" == "/etc/crowdsec/config.yaml" ] ; then
config_exists=$(conf_get '.api.server.online_client | has("credentials_path")')
if isfalse "$config_exists"; then
conf_set '.api.server.online_client = {"credentials_path": "/etc/crowdsec/online_api_credentials.yaml"}'
cscli capi register > /etc/crowdsec/online_api_credentials.yaml
echo "Registration to online API done"
fi
fi
## Enroll instance if enroll key is provided
if [ "${DISABLE_ONLINE_API,,}" != "true" ] && [ "$ENROLL_KEY" != "" ] ; then
# Enroll instance if enroll key is provided
if isfalse "$DISABLE_ONLINE_API" && [ "$ENROLL_KEY" != "" ]; then
enroll_args=""
if [ "$ENROLL_INSTANCE_NAME" != "" ] ; then
if [ "$ENROLL_INSTANCE_NAME" != "" ]; then
enroll_args="--name $ENROLL_INSTANCE_NAME"
fi
if [ "$ENROLL_TAGS" != "" ] ; then
#shellcheck disable=SC2086
for tag in ${ENROLL_TAGS}
do
if [ "$ENROLL_TAGS" != "" ]; then
# shellcheck disable=SC2086
for tag in ${ENROLL_TAGS}; do
enroll_args="$enroll_args --tags $tag"
done
fi
#shellcheck disable=SC2086
# shellcheck disable=SC2086
cscli console enroll $enroll_args "$ENROLL_KEY"
fi
# crowdsec sqlite database permissions
if [ "$GID" != "" ]; then
IS_SQLITE=$(yq eval '.db_config.type == "sqlite"' "$CS_CONFIG_FILE")
DB_PATH=$(yq eval '.db_config.db_path' "$CS_CONFIG_FILE")
if [ "$IS_SQLITE" == "true" ]; then
chown ":$GID" "$DB_PATH"
if istrue "$(conf_get '.db_config.type == "sqlite"')"; then
chown ":$GID" "$(conf_get '.db_config.db_path')"
echo "sqlite database permissions updated"
fi
fi
if [ "${USE_TLS,,}" == "true" ]; then
yq -i eval ".api.server.tls.cert_file = \"$CERT_FILE\"" "$CS_CONFIG_FILE"
yq -i eval ".api.server.tls.key_file = \"$KEY_FILE\"" "$CS_CONFIG_FILE"
yq -i eval '... comments=""' "$CS_CONFIG_FILE"
if istrue "$USE_TLS"; then
agents_allowed_yaml=$(csv2yaml "$AGENTS_ALLOWED_OU") \
bouncers_allowed_yaml=$(csv2yaml "$BOUNCERS_ALLOWED_OU") \
conf_set '
.api.server.tls.ca_cert_path = strenv(CACERT_FILE) |
.api.server.tls.cert_file = strenv(CERT_FILE) |
.api.server.tls.key_file = strenv(KEY_FILE) |
.api.server.tls.bouncers_allowed_ou = env(bouncers_allowed_yaml) |
.api.server.tls.agents_allowed_ou = env(agents_allowed_yaml) |
... comments=""
'
fi
if [ "$PLUGIN_DIR" != "/usr/local/lib/crowdsec/plugins/" ]; then
yq -i eval ".config_paths.plugin_dir = \"$PLUGIN_DIR\"" "$CS_CONFIG_FILE"
fi
conf_set ".config_paths.plugin_dir = strenv(PLUGIN_DIR)"
## Install collections, parsers, scenarios & postoverflows
cscli -c "$CS_CONFIG_FILE" hub update
cscli -c "$CS_CONFIG_FILE" collections upgrade crowdsecurity/linux || true
cscli -c "$CS_CONFIG_FILE" parsers upgrade crowdsecurity/whitelists || true
cscli -c "$CS_CONFIG_FILE" parsers install crowdsecurity/docker-logs || true
cscli hub update
cscli collections upgrade crowdsecurity/linux || true
cscli parsers upgrade crowdsecurity/whitelists || true
cscli parsers install crowdsecurity/docker-logs || true
if [ "$COLLECTIONS" != "" ]; then
#shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" collections install $COLLECTIONS
# shellcheck disable=SC2086
cscli collections install $COLLECTIONS
fi
if [ "$PARSERS" != "" ]; then
#shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" parsers install $PARSERS
# shellcheck disable=SC2086
cscli parsers install $PARSERS
fi
if [ "$SCENARIOS" != "" ]; then
#shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" scenarios install $SCENARIOS
# shellcheck disable=SC2086
cscli scenarios install $SCENARIOS
fi
if [ "$POSTOVERFLOWS" != "" ]; then
#shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" postoverflows install $POSTOVERFLOWS
# shellcheck disable=SC2086
cscli postoverflows install $POSTOVERFLOWS
fi
## Remove collections, parsers, scenarios & postoverflows
if [ "$DISABLE_COLLECTIONS" != "" ]; then
#shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" collections remove $DISABLE_COLLECTIONS
fi
if [ "$DISABLE_PARSERS" != "" ]; then
#shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" parsers remove $DISABLE_PARSERS
fi
if [ "$DISABLE_SCENARIOS" != "" ]; then
#shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" scenarios remove $DISABLE_SCENARIOS
fi
if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
#shellcheck disable=SC2086
cscli -c "$CS_CONFIG_FILE" postoverflows remove $DISABLE_POSTOVERFLOWS
# shellcheck disable=SC2086
cscli collections remove $DISABLE_COLLECTIONS
fi
function register_bouncer {
if ! cscli -c "$CS_CONFIG_FILE" bouncers list -o json | sed '/^ *"name"/!d;s/^ *"name": "\(.*\)",/\1/' | grep -q "^${NAME}$"; then
if cscli -c "$CS_CONFIG_FILE" bouncers add "${NAME}" -k "${KEY}" > /dev/null; then
if [ "$DISABLE_PARSERS" != "" ]; then
# shellcheck disable=SC2086
cscli parsers remove $DISABLE_PARSERS
fi
if [ "$DISABLE_SCENARIOS" != "" ]; then
# shellcheck disable=SC2086
cscli scenarios remove $DISABLE_SCENARIOS
fi
if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
# shellcheck disable=SC2086
cscli postoverflows remove $DISABLE_POSTOVERFLOWS
fi
register_bouncer() {
if ! cscli bouncers list -o json | sed '/^ *"name"/!d;s/^ *"name": "\(.*\)",/\1/' | grep -q "^${NAME}$"; then
if cscli bouncers add "${NAME}" -k "${KEY}" > /dev/null; then
echo "Registered bouncer for ${NAME}"
else
echo "Failed to register bouncer for ${NAME}"
@ -182,6 +247,7 @@ ARGS=""
if [ "$CONFIG_FILE" != "" ]; then
ARGS="-c $CONFIG_FILE"
fi
if [ "$DSN" != "" ]; then
ARGS="$ARGS -dsn ${DSN}"
fi
@ -189,24 +255,32 @@ fi
if [ "$TYPE" != "" ]; then
ARGS="$ARGS -type $TYPE"
fi
if [ "${TEST_MODE,,}" == "true" ]; then
if istrue "$TEST_MODE"; then
ARGS="$ARGS -t"
fi
if [ "${DISABLE_AGENT,,}" == "true" ]; then
if istrue "$DISABLE_AGENT"; then
ARGS="$ARGS -no-cs"
fi
if [ "${DISABLE_LOCAL_API,,}" == "true" ]; then
if istrue "$DISABLE_LOCAL_API"; then
ARGS="$ARGS -no-api"
fi
if [ "${LEVEL_TRACE,,}" == "true" ]; then
if istrue "$LEVEL_TRACE"; then
ARGS="$ARGS -trace"
fi
if [ "${LEVEL_DEBUG,,}" == "true" ]; then
if istrue "$LEVEL_DEBUG"; then
ARGS="$ARGS -debug"
fi
if [ "${LEVEL_INFO,,}" == "true" ]; then
if istrue "$LEVEL_INFO"; then
ARGS="$ARGS -info"
fi
#shellcheck disable=SC2086
conf_set '.prometheus.listen_port=env(METRICS_PORT)'
# shellcheck disable=SC2086
exec crowdsec $ARGS

4
pkg/apiserver/apiserver.go
View file

@ -322,11 +322,11 @@ func (s *APIServer) Run(apiReady chan bool) error {
log.Infof("CrowdSec Local API listening on %s", s.URL)
if s.TLS != nil && s.TLS.CertFilePath != "" && s.TLS.KeyFilePath != "" {
if err := s.httpServer.ListenAndServeTLS(s.TLS.CertFilePath, s.TLS.KeyFilePath); err != nil {
log.Fatal(err)
log.Fatalf("while serving local API: %v", err)
}
} else {
if err := s.httpServer.ListenAndServe(); err != http.ErrServerClosed {
log.Fatal(err)
log.Fatalf("while serving local API: %v", err)
}
}
}()