non-fatal error if some datasource can't be run (i.e. journalctl but systemd is missing) (#2310)

This on the other hand, gives a new fatal error when there are no valid datasources.
In the previous version, crowdsec kept running with just a warning if no
acquisition yaml or dir were specified.

.dockerignore

@@ -1,3 +1,4 @@
 # We include .git in the build context because excluding it would break the
 # "make release" target, which uses git to retrieve the build version and tag.
 #.git
+/tests/ansible

.github/workflows/ci-windows-build-msi.yml

@@ -1,17 +1,11 @@
 name: build-msi (windows)
 
 on:
-  pull_request:
+  release:
-    branches:
+    types:
-      - master
+      - prereleased
-      - releases/**
-    paths-ignore:
-      - 'docs/**'
-      - 'mkdocs.yml'
-      - 'README.md'
 
 jobs:
-
   build:
     name: Build
     runs-on: windows-2019
@@ -22,14 +16,9 @@ jobs:
         go-version: 1.19
       id: go
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
-    - id: get_latest_release
-      uses: pozetroninc/github-action-get-latest-release@master
       with:
-        repository: crowdsecurity/crowdsec
+        fetch-depth: 0
-        excludes: draft
-    - id: set_release_in_env
-      run: echo "BUILD_VERSION=${{ steps.get_latest_release.outputs.release }}" >> $env:GITHUB_ENV
     - name: Build
       run: make windows_installer
     - name: Upload MSI

.github/workflows/dispatch_ci_hub.yaml

@@ -1,22 +0,0 @@
-name: Dispatch to hub when creating pre-release
-
-on:
-  push:
-    # branches to consider in the event; optional, defaults to all
-    branches:
-      - master
-      - releases/**
-
-jobs:
-  dispatch:
-    name: dispatch to hub-tests
-    runs-on: ubuntu-latest
-    steps:
-      - name: Repository Dispatch
-        uses: peter-evans/repository-dispatch@v1
-        if: ${{ github.repository_owner == 'crowdsecurity' }}
-        with:
-          token: ${{ secrets.DISPATCH_TOKEN }}
-          event-type: trigger_ci_hub
-          repository: crowdsecurity/hub
-          client-payload: '{"version": "${{ steps.keydb.outputs.release }}"}'

.github/workflows/dispatch_create_branch_hub.yaml

@@ -1,24 +0,0 @@
-name: Dispatch to hub when creating pre-release
-
-on:
-  release:
-    types: prereleased
-
-jobs:
-  dispatch:
-    name: dispatch to hub-tests
-    runs-on: ubuntu-latest
-    steps:
-      - id: keydb
-        uses: pozetroninc/github-action-get-latest-release@master
-        with:
-          owner: crowdsecurity
-          repo: crowdsec
-          excludes: prerelease, draft
-      - name: Repository Dispatch
-        uses: peter-evans/repository-dispatch@v1
-        with:
-          token: ${{ secrets.DISPATCH_TOKEN }}
-          event-type: create_branch
-          repository: crowdsecurity/hub
-          client-payload: '{"version": "${{ steps.keydb.outputs.release }}"}'

.github/workflows/dispatch_delete_branch_hub.yaml

@@ -1,24 +0,0 @@
-name: Dispatch to hub when deleting pre-release
-
-on:
-  release:
-    types: deleted
-
-jobs:
-  dispatch:
-    name: dispatch to hub-tests
-    runs-on: ubuntu-latest
-    steps:
-      - id: keydb
-        uses: pozetroninc/github-action-get-latest-release@master
-        with:
-          owner: crowdsecurity
-          repo: crowdsec
-          excludes: prerelease, draft
-      - name: Repository Dispatch
-        uses: peter-evans/repository-dispatch@v1
-        with:
-          token: ${{ secrets.DISPATCH_TOKEN }}
-          event-type: delete_branch
-          repository: crowdsecurity/hub
-          client-payload: '{"version": "${{ steps.keydb.outputs.release }}"}'

.github/workflows/release_publish-package.yml

@@ -3,7 +3,8 @@ name: build
 
 on:
   release:
-    types: prereleased
+    types:
+      - prereleased
 
 jobs:
   build:
@@ -17,6 +18,8 @@ jobs:
         id: go
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
       - name: Build the binaries
         run: make release
       - name: Upload to release
@@ -36,6 +39,8 @@ jobs:
         id: go
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
       - name: Build the binaries
         run: |
           make release BUILD_STATIC=yes

.github/workflows/release_publish_docker-image.yml

@@ -32,7 +32,7 @@ jobs:
           TAGS_SLIM="${DOCKER_IMAGE}:${VERSION}-slim"
           if [[ ${{ github.event.action }} == released ]]; then
             TAGS=$TAGS,${DOCKER_IMAGE}:latest,${GHCR_IMAGE}:latest
-            TAGS_SLIM=$TAGS,${DOCKER_IMAGE}:slim
+            TAGS_SLIM=$TAGS_SLIM,${DOCKER_IMAGE}:slim
           fi
           echo ::set-output name=version::${VERSION}
           echo ::set-output name=tags::${TAGS}

Dockerfile

@@ -1,3 +1,4 @@
+# vim: set ft=dockerfile:
 ARG BUILD_ENV=full
 ARG GOVERSION=1.19
 
@@ -9,82 +10,30 @@ COPY . .
 
 # wizard.sh requires GNU coreutils
 RUN apk add --no-cache git gcc libc-dev make bash gettext binutils-gold coreutils && \
+    echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
     SYSTEM="docker" make clean release && \
     cd crowdsec-v* && \
     ./wizard.sh --docker-mode && \
-    cd - && \
+    cd - >/dev/null && \
     cscli hub update && \
     cscli collections install crowdsecurity/linux && \
-    cscli parsers install crowdsecurity/whitelists
+    cscli parsers install crowdsecurity/whitelists && \
+    go install github.com/mikefarah/yq/v4@v4.30.6
 
 FROM alpine:latest as build-slim
 
-RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata yq bash && \
+RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata bash && \
     mkdir -p /staging/etc/crowdsec && \
     mkdir -p /staging/var/lib/crowdsec && \
     mkdir -p /var/lib/crowdsec/data
 
+COPY --from=build /go/bin/yq /usr/local/bin/yq
 COPY --from=build /etc/crowdsec /staging/etc/crowdsec
 COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
 COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
 COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
 COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
-
+RUN yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml
-ENV CONFIG_FILE=/etc/crowdsec/config.yaml
-ENV LOCAL_API_URL=http://0.0.0.0:8080/
-ENV CUSTOM_HOSTNAME=localhost
-ENV PLUGIN_DIR=/usr/local/lib/crowdsec/plugins/
-ENV DISABLE_AGENT=false
-ENV DISABLE_LOCAL_API=false
-ENV DISABLE_ONLINE_API=false
-ENV DSN=
-ENV TYPE=
-ENV TEST_MODE=false
-ENV USE_WAL=false
-
-# register to app.crowdsec.net
-
-ENV ENROLL_INSTANCE_NAME=
-ENV ENROLL_KEY=
-ENV ENROLL_TAGS=
-
-# log verbosity
-
-ENV LEVEL_TRACE=false
-ENV LEVEL_DEBUG=false
-ENV LEVEL_INFO=true
-
-# TLS setup ----------------------------------- #
-
-ENV AGENT_USERNAME=
-ENV AGENT_PASSWORD=
-
-# TLS setup ----------------------------------- #
-
-ENV USE_TLS=false
-ENV CA_CERT_PATH=
-ENV CERT_FILE=/etc/ssl/cert.pem
-ENV KEY_FILE=/etc/ssl/key.pem
-# comma-separated list of allowed OU values for TLS bouncer certificates
-ENV BOUNCERS_ALLOWED_OU=bouncer-ou
-# comma-separated list of allowed OU values for TLS agent certificates
-ENV AGENTS_ALLOWED_OU=agent-ou
-
-# Install the following hub items --------------#
-
-ENV COLLECTIONS=
-ENV PARSERS=
-ENV SCENARIOS=
-ENV POSTOVERFLOWS=
-
-# Uninstall the following hub items ------------#
-
-ENV DISABLE_COLLECTIONS=
-ENV DISABLE_PARSERS=
-ENV DISABLE_SCENARIOS=
-ENV DISABLE_POSTOVERFLOWS=
-
-ENV METRICS_PORT=6060
 
 ENTRYPOINT /bin/bash docker_start.sh
 

Dockerfile.debian

@@ -8,20 +8,27 @@ WORKDIR /go/src/crowdsec
 
 COPY . .
 
+ENV DEBIAN_FRONTEND=noninteractive
+ENV DEBCONF_NOWARNINGS="yes"
+
 # wizard.sh requires GNU coreutils
 RUN apt-get update && \
-    apt-get install -y git gcc libc-dev make bash gettext binutils-gold coreutils tzdata && \
+    apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata && \
-    SYSTEM="docker" make release && \
+    echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
+    SYSTEM="docker" make clean release && \
     cd crowdsec-v* && \
     ./wizard.sh --docker-mode && \
-    cd - && \
+    cd - >/dev/null && \
     cscli hub update && \
     cscli collections install crowdsecurity/linux && \
     cscli parsers install crowdsecurity/whitelists && \
-    go install github.com/mikefarah/yq/v4@latest
+    go install github.com/mikefarah/yq/v4@v4.30.6
 
 FROM debian:bullseye-slim as build-slim
 
+ENV DEBIAN_FRONTEND=noninteractive
+ENV DEBCONF_NOWARNINGS="yes"
+
 RUN apt-get update && \
     apt-get install -y -q --install-recommends --no-install-suggests \
     procps \
@@ -40,63 +47,9 @@ COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
 COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
 COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
 COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
-RUN yq eval -i ".plugin_config.group = \"nogroup\"" /staging/etc/crowdsec/config.yaml
+RUN yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml && \
+    yq eval -i ".plugin_config.group = \"nogroup\"" /staging/etc/crowdsec/config.yaml
 
-ENV CONFIG_FILE=/etc/crowdsec/config.yaml
-ENV LOCAL_API_URL=http://0.0.0.0:8080/
-ENV CUSTOM_HOSTNAME=localhost
-ENV PLUGIN_DIR=/usr/local/lib/crowdsec/plugins/
-ENV DISABLE_AGENT=false
-ENV DISABLE_LOCAL_API=false
-ENV DISABLE_ONLINE_API=false
-ENV DSN=
-ENV TYPE=
-ENV TEST_MODE=false
-ENV USE_WAL=false
-
-# register to app.crowdsec.net
-
-ENV ENROLL_INSTANCE_NAME=
-ENV ENROLL_KEY=
-ENV ENROLL_TAGS=
-
-# log verbosity
-
-ENV LEVEL_TRACE=false
-ENV LEVEL_DEBUG=false
-ENV LEVEL_INFO=true
-
-# TLS setup ----------------------------------- #
-
-ENV AGENT_USERNAME=
-ENV AGENT_PASSWORD=
-
-# TLS setup ----------------------------------- #
-
-ENV USE_TLS=false
-ENV CA_CERT_PATH=
-ENV CERT_FILE=/etc/ssl/cert.pem
-ENV KEY_FILE=/etc/ssl/key.pem
-# comma-separated list of allowed OU values for TLS bouncer certificates
-ENV BOUNCERS_ALLOWED_OU=bouncer-ou
-# comma-separated list of allowed OU values for TLS agent certificates
-ENV AGENTS_ALLOWED_OU=agent-ou
-
-# Install the following hub items --------------#
-
-ENV COLLECTIONS=
-ENV PARSERS=
-ENV SCENARIOS=
-ENV POSTOVERFLOWS=
-
-# Uninstall the following hub items ------------#
-
-ENV DISABLE_COLLECTIONS=
-ENV DISABLE_PARSERS=
-ENV DISABLE_SCENARIOS=
-ENV DISABLE_POSTOVERFLOWS=
-
-ENV METRICS_PORT=6060
 
 ENTRYPOINT /bin/bash docker_start.sh
 

azure-pipelines.yml

@@ -56,25 +56,26 @@ stages:
               --input "**/*.exe" --config (Join-Path -Path $(Agent.TempDirectory) -ChildPath "appsettings.json") `
               --user $(CodeSigningUser) --secret '$(CodeSigningPassword)'
             displayName: "Sign Crowdsec binaries + plugins"
-
           - pwsh: |
-              $build_version=(git describe --tags (git rev-list --tags --max-count=1)).Substring(1)
+              $build_version=$env:BUILD_SOURCEBRANCHNAME
+              if ($build_version.StartsWith("v"))
+              {
+                $build_version = $build_version.Substring(1)
+              }
               if ($build_version.Contains("-")) 
               {
                 $build_version = $build_version.Substring(0, $build_version.IndexOf("-"))
               }
-              .\make_installer.ps1 -version $build_version
               Write-Host  "##vso[task.setvariable variable=BuildVersion;isOutput=true]$build_version"
+            displayName: GetCrowdsecVersion
+            name: GetCrowdsecVersion
+          - pwsh: |
+              .\make_installer.ps1 -version '$(GetCrowdsecVersion.BuildVersion)'
             displayName: "Build Crowdsec MSI"
             name: BuildMSI
 
           - pwsh: |
-              $build_version=(git describe --tags (git rev-list --tags --max-count=1)).Substring(1)
+              .\make_chocolatey.ps1 -version '$(GetCrowdsecVersion.BuildVersion)'
-              if ($build_version.Contains("-")) 
-              {
-                $build_version = $build_version.Substring(0, $build_version.IndexOf("-"))
-              }
-              .\make_chocolatey.ps1 -version $build_version
             displayName: "Build Chocolatey nupkg"
 
           - pwsh: |
@@ -85,14 +86,14 @@ stages:
 
           - task: PublishBuildArtifacts@1
             inputs:
-              PathtoPublish: '$(Build.Repository.LocalPath)\\crowdsec_$(BuildMSI.BuildVersion).msi'
+              PathtoPublish: '$(Build.Repository.LocalPath)\\crowdsec_$(GetCrowdsecVersion.BuildVersion).msi'
               ArtifactName: 'crowdsec.msi'
               publishLocation: 'Container'
             displayName: "Upload MSI artifact"
           
           - task: PublishBuildArtifacts@1
             inputs:
-              PathtoPublish: '$(Build.Repository.LocalPath)\\windows\\Chocolatey\\crowdsec\\crowdsec.$(BuildMSI.BuildVersion).nupkg'
+              PathtoPublish: '$(Build.Repository.LocalPath)\\windows\\Chocolatey\\crowdsec\\crowdsec.$(GetCrowdsecVersion.BuildVersion).nupkg'
               ArtifactName: 'crowdsec.nupkg'
               publishLocation: 'Container'
             displayName: "Upload nupkg artifact"

cmd/crowdsec-cli/main.go

@@ -52,7 +52,7 @@ func initConfig() {
 	} else if err_lvl {
 		log.SetLevel(log.ErrorLevel)
 	}
-	logFormatter := &log.TextFormatter{TimestampFormat: "02-01-2006 03:04:05 PM", FullTimestamp: true}
+	logFormatter := &log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}
 	log.SetFormatter(logFormatter)
 
 	if !inSlice(os.Args[1], NoNeedConfig) {

cmd/crowdsec/crowdsec.go

@@ -23,22 +23,22 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
 	var err error
 
 	// Populate cwhub package tools
-	if err := cwhub.GetHubIdx(cConfig.Hub); err != nil {
+	if err = cwhub.GetHubIdx(cConfig.Hub); err != nil {
-		return &parser.Parsers{}, fmt.Errorf("Failed to load hub index : %s", err)
+		return &parser.Parsers{}, fmt.Errorf("while loading hub index : %s", err)
 	}
 
 	// Start loading configs
 	csParsers := newParsers()
 	if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
-		return &parser.Parsers{}, fmt.Errorf("Failed to load parsers: %s", err)
+		return nil, fmt.Errorf("while loading parsers: %s", err)
 	}
 
 	if err := LoadBuckets(cConfig); err != nil {
-		return &parser.Parsers{}, fmt.Errorf("Failed to load scenarios: %s", err)
+		return nil, fmt.Errorf("while loading scenarios: %s", err)
 	}
 
 	if err := LoadAcquisition(cConfig); err != nil {
-		return &parser.Parsers{}, fmt.Errorf("Error while loading acquisition config : %s", err)
+		return nil, fmt.Errorf("while loading acquisition config: %s", err)
 	}
 	return csParsers, nil
 }

cmd/crowdsec/main.go

@@ -157,6 +157,10 @@ func LoadAcquisition(cConfig *csconfig.Config) error {
 		}
 	}
 
+	if len(dataSources) == 0 {
+		return fmt.Errorf("no datasource enabled")
+	}
+
 	return nil
 }
 

docker/README.md

@@ -138,10 +138,18 @@ agents on each machine that runs the protected applications, and a LAPI that
 gathers all signals from agents and communicates with the `central API`.
 
 ## Register a new agent with LAPI
+
+Without TLS authentication:
+
 ```shell
 docker exec -it crowdsec_lapi_container_name cscli machines add agent_user_name --password agent_password
 ```
 
+With TLS authentication:
+
+Agents are automatically registered and don't need a username or password. The
+agents' names are derived from the IP address from which they connect.
+
 ## Run an agent connected to LAPI
 
 Add the following environment variables to the docker run command:
@@ -163,13 +171,20 @@ https://docs.crowdsec.net/docs/user_guides/bouncers_configuration/
 
 ### Automatic Bouncer Registration
 
-You can automatically register bouncers with the crowdsec container at startup, using environment variables or Docker secrets. You cannot use this process to update an existing bouncer without first deleting it.
+Without TLS authentication:
+
+You can register bouncers with the crowdsec container at startup, using environment variables or Docker secrets. You cannot use this process to update an existing bouncer without first deleting it.
 
 To use environment variables, they should be in the format `BOUNCER_KEY_<name>=<key>`. e.g. `BOUNCER_KEY_nginx=mysecretkey12345`.
 
 To use Docker secrets, the secret should be named `bouncer_key_<name>` with a content of `<key>`. e.g. `bouncer_key_nginx` with content `mysecretkey12345`.
 
-A bouncer key can be any string but we recommend an alphanumeric value for consistency with crowdsec-generated keys and avoid problems with escaping special characters.
+A bouncer key can be any string but we recommend an alphanumeric value for consistency with the crowdsec-generated keys and to avoid problems with escaping special characters.
+
+With TLS authentication:
+
+Bouncers are automatically registered and don't need an API key. The
+bouncers' names are derived from the IP address from which they connect.
 
 ## Console
 We provide a web-based interface to get more from Crowdsec: https://docs.crowdsec.net/docs/console
@@ -183,22 +198,33 @@ Using binds rather than named volumes ([complete explanation here](https://docs.
 # Reference
 ## Environment Variables
 
+Note for persistent configurations (i.e. bind mount or volumes): when a
+variable is set, its value may be written to the appropriate file (usually
+config.yaml) each time the container is run.
+
+
 | Variable                | Default                   | Description |
 | ----------------------- | ------------------------- | ----------- |
 | `CONFIG_FILE`           | `/etc/crowdsec/config.yaml` | Configuration file location |
-| `DSN`                   | | Process a single source in time-machine: `-e DSN="file:///var/log/toto.log"` or `-e DSN="cloudwatch:///your/group/path:stream_name?profile=dev&backlog=16h"` or `-e DSN="journalctl://filters=_SYSTEMD_UNIT=ssh.service"` |
-| `TYPE`                  | | [`Labels.type`](https://docs.crowdsec.net/Crowdsec/v1/references/acquisition/) for file in time-machine: `-e TYPE="<type>"` |
-| `TEST_MODE`             | false | Don't run the service, only test the configuration: `-e TEST_MODE=true` |
-| `TZ`                    | | Set the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) to ensure the logs have a local timestamp. |
-| `LOCAL_API_URL`         | `http://0.0.0.0:8080` | The LAPI URL, you need to change this when `DISABLE_LOCAL_API` is true: `-e LOCAL_API_URL="http://lapi-address:8080"` |
 | `DISABLE_AGENT`         | false | Disable the agent, run a LAPI-only container |
 | `DISABLE_LOCAL_API`     | false | Disable LAPI, run an agent-only container |
 | `DISABLE_ONLINE_API`    | false | Disable online API registration for signal sharing |
-| `CUSTOM_HOSTNAME`       | localhost | Custom hostname for LAPI registration (with agent and LAPI on the same container) |
+| `TEST_MODE`             | false | Don't run the service, only test the configuration: `-e TEST_MODE=true` |
+| `TZ`                    | | Set the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) to ensure the logs have a local timestamp. |
+| `LOCAL_API_URL`         | `http://0.0.0.0:8080` | The LAPI URL, you need to change this when `DISABLE_LOCAL_API` is true: `-e LOCAL_API_URL="http://lapi-address:8080"` |
 | `PLUGIN_DIR`            | `/usr/local/lib/crowdsec/plugins/` | Directory for plugins: `-e PLUGIN_DIR="<path>"` |
-| `BOUNCER_KEY_<name>`    | | Register a bouncer with the name `<name>` and a key equal to the value of the environment variable. |
 | `METRICS_PORT`          | 6060 | Port to expose Prometheus metrics |
+|                         | | |
+| __LAPI__                | | (useless with DISABLE_LOCAL_API) |
 | `USE_WAL`               | false | Enable Write-Ahead Logging with SQLite |
+| `CUSTOM_HOSTNAME`       | localhost | Name for the local agent (running in the container with LAPI) |
+|                         | | |
+| __Agent__               | | (these don't work with DISABLE_AGENT) |
+| `TYPE`                  | | [`Labels.type`](https://docs.crowdsec.net/Crowdsec/v1/references/acquisition/) for file in time-machine: `-e TYPE="<type>"` |
+| `DSN`                   | | Process a single source in time-machine: `-e DSN="file:///var/log/toto.log"` or `-e DSN="cloudwatch:///your/group/path:stream_name?profile=dev&backlog=16h"` or `-e DSN="journalctl://filters=_SYSTEMD_UNIT=ssh.service"` |
+|                         | | |
+| __Bouncers__            | | |
+| `BOUNCER_KEY_<name>`    | | Register a bouncer with the name `<name>` and a key equal to the value of the environment variable. |
 |                         | | |
 | __Console__             | | |
 | `ENROLL_KEY`            | | Enroll key retrieved from [the console](https://app.crowdsec.net/) to enroll the instance. |
@@ -209,18 +235,23 @@ Using binds rather than named volumes ([complete explanation here](https://docs.
 | `AGENT_USERNAME`        | | Agent username (to register if is LAPI or to use if it's an agent): `-e AGENT_USERNAME="machine_id"` |
 | `AGENT_PASSWORD`        | | Agent password (to register if is LAPI or to use if it's an agent): `-e AGENT_PASSWORD="machine_password"` |
 |                         | | |
-| __TLS Auth/encryption   | | |
+| __TLS Encryption__      | | |
-| `USE_TLS`               | false | Enable TLS on the LAPI |
+| `USE_TLS`               | false | Enable TLS encryption (either as a LAPI or agent) |
-| `CERT_FILE`             | /etc/ssl/cert.pem | TLS Certificate path |
+| `CACERT_FILE`           | | CA certificate bundle (for self-signed certificates) |
-| `KEY_FILE`              | /etc/ssl/key.pem | TLS Key path |
+| `INSECURE_SKIP_VERIFY`  | | Skip LAPI certificate validation |
-| `CACERT_FILE`           | | CA certificate |
+| `LAPI_CERT_FILE`        | | LAPI TLS Certificate path |
+| `LAPI_KEY_FILE`         | | LAPI TLS Key path |
+|                         | | |
+| __TLS Authentication__  | | (these require USE_TLS=true) |
+| `CLIENT_CERT_FILE`      | | Client TLS Certificate path (enable TLS authentication) |
+| `CLIENT_KEY_FILE`       | | Client TLS Key path |
 | `AGENTS_ALLOWED_OU`     | agent-ou | OU values allowed for agents, separated by comma |
 | `BOUNCERS_ALLOWED_OU`   | bouncer-ou | OU values allowed for bouncers, separated by comma |
 |                         | | |
 | __Hub management__      | | |
 | `COLLECTIONS`           | | Collections to install, separated by space: `-e COLLECTIONS="crowdsecurity/linux crowdsecurity/apache2"` |
-| `SCENARIOS`             | | Scenarios to install, separated by space |
 | `PARSERS`               | | Parsers to install, separated by space |
+| `SCENARIOS`             | | Scenarios to install, separated by space |
 | `POSTOVERFLOWS`         | | Postoverflows to install, separated by space |
 | `DISABLE_COLLECTIONS`   | | Collections to remove, separated by space: `-e DISABLE_COLLECTIONS="crowdsecurity/linux crowdsecurity/nginx"` |
 | `DISABLE_PARSERS`       | | Parsers to remove, separated by space |
@@ -231,6 +262,10 @@ Using binds rather than named volumes ([complete explanation here](https://docs.
 | `LEVEL_INFO`            | false | Force INFO level for the container log |
 | `LEVEL_DEBUG`           | false | Force DEBUG level for the container log |
 | `LEVEL_TRACE`           | false | Force TRACE level (VERY verbose) for the container log |
+|                         | | |
+| __Developer options__   | | |
+| `CI_TESTING`            | false | Used during functional tests |
+| `DEBUG`                 | false | Trace the entrypoint |
 
 ## Volumes
 

docker/config.yaml

@@ -25,14 +25,10 @@ db_config:
   log_level: info
   type: sqlite
   db_path: /var/lib/crowdsec/data/crowdsec.db
-  #user: 
-  #password:
-  #db_name:
-  #host:
-  #port:
   flush:
     max_items: 5000
     max_age: 7d
+  use_wal: false
 api:
   client:
     insecure_skip_verify: false
@@ -46,9 +42,11 @@ api:
       - ::1
     online_client: # Central API credentials (to push signals and receive bad IPs)
     #credentials_path: /etc/crowdsec/online_api_credentials.yaml
-#    tls:
+    tls:
-#      cert_file: /etc/crowdsec/ssl/cert.pem
+      agents_allowed_ou:
-#      key_file: /etc/crowdsec/ssl/key.pem
+        - agent-ou
+      bouncers_allowed_ou:
+        - bouncer-ou
 prometheus:
   enabled: true
   level: full

docker/docker_start.sh

@@ -6,8 +6,7 @@
 set -e
 shopt -s inherit_errexit
 
-#- HELPER FUNCTIONS ----------------#
+# match true, TRUE, True, tRuE, etc.
-
 istrue() {
   case "$(echo "$1" | tr '[:upper:]' '[:lower:]')" in
     true) return 0 ;;
@@ -23,6 +22,23 @@ isfalse() {
     fi
 }
 
+if istrue "$DEBUG"; then
+    set -x
+    export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
+fi
+
+if istrue "$CI_TESTING"; then
+    echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" >/etc/machine-id
+fi
+
+#- DEFAULTS -----------------------#
+
+export CONFIG_FILE="${CONFIG_FILE:=/etc/crowdsec/config.yaml}"
+export CUSTOM_HOSTNAME="${CUSTOM_HOSTNAME:=localhost}"
+
+#- HELPER FUNCTIONS ----------------#
+
+# csv2yaml <string>
 # generate a yaml list from a comma-separated string of values
 csv2yaml() {
     [ -z "$1" ] && return
@@ -34,6 +50,8 @@ cscli() {
     command cscli -c "$CONFIG_FILE" "$@"
 }
 
+# conf_get <key> [file_path]
+# retrieve a value from a file (by default $CONFIG_FILE)
 conf_get() {
     if [ $# -ge 2 ]; then
         yq e "$1" "$2"
@@ -42,16 +60,68 @@ conf_get() {
     fi
 }
 
+# conf_set <yq_expression> [file_path]
+# evaluate a yq command (by default on $CONFIG_FILE),
+# create the file if it doesn't exist
 conf_set() {
     if [ $# -ge 2 ]; then
-        yq e "$1" -i "$2"
+        YAML_FILE="$2"
     else
-        yq e "$1" -i "$CONFIG_FILE"
+        YAML_FILE="$CONFIG_FILE"
+    fi
+    YAML_CONTENT=$(cat "$YAML_FILE" 2>/dev/null || true)
+    NEW_CONTENT=$(echo "$YAML_CONTENT" | yq e "$1")
+    echo "$NEW_CONTENT" | install -m 0600 /dev/stdin "$YAML_FILE"
+}
+
+# conf_set_if(): used to update the configuration
+# only if a given variable is provided
+# conf_set_if "$VAR" <yq_expression> [file_path]
+conf_set_if() {
+    if [ "$1" != "" ]; then
+        shift
+        conf_set "$@"
     fi
 }
 
+# register_bouncer <bouncer_name> <bouncer_key>
+register_bouncer() {
+  if ! cscli bouncers list -o json | sed '/^ *"name"/!d;s/^ *"name": "\(.*\)",/\1/' | grep -q "^${1}$"; then
+      if cscli bouncers add "$1" -k "$2" > /dev/null; then
+          echo "Registered bouncer for $1"
+      else
+          echo "Failed to register bouncer for $1"
+      fi
+  fi
+}
+
+# Call cscli to manage objects ignoring taint errors
+# $1 can be collections, parsers, etc.
+# $2 can be install, remove, upgrade
+# $3 is a list of object names separated by space
+cscli_if_clean() {
+    # loop over all objects
+    for obj in $3; do
+        if cscli "$1" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then
+            echo "Object $1/$obj is tainted, skipping"
+        else
+            cscli "$1" "$2" "$obj"
+        fi
+    done
+}
+
 #-----------------------------------#
 
+if [ -n "$CERT_FILE" ] || [ -n "$KEY_FILE" ] ; then
+    printf '%b' '\033[0;33m'
+    echo "Warning: the variables CERT_FILE and KEY_FILE have been deprecated." >&2
+    echo "Please use LAPI_CERT_FILE and LAPI_KEY_FILE insted." >&2
+    echo "The old variables will be removed in a future release." >&2
+    printf '%b' '\033[0m'
+    LAPI_CERT_FILE=${LAPI_CERT_FILE:-$CERT_FILE}
+    LAPI_KEY_FILE=${LAPI_KEY_FILE:-$KEY_FILE}
+fi
+
 # Check and prestage databases
 for geodb in GeoLite2-ASN.mmdb GeoLite2-City.mmdb; do
     # We keep the pre-populated geoip databases in /staging instead of /var,
@@ -84,52 +154,65 @@ elif [ -n "$USE_WAL" ] && isfalse "$USE_WAL"; then
     conf_set '.db_config.use_wal = false'
 fi
 
-# regenerate local agent credentials (ignore if agent is disabled)
+# regenerate local agent credentials (even if agent is disabled, cscli needs a
-if isfalse "$DISABLE_AGENT"; then
+# connection to the API)
-    if isfalse "$DISABLE_LOCAL_API"; then
-        echo "Regenerate local agent credentials"
 cscli machines delete "$CUSTOM_HOSTNAME" 2>/dev/null || true
-        # shellcheck disable=SC2086
-        cscli machines add "$CUSTOM_HOSTNAME" --auto --url "$LOCAL_API_URL"
-    fi
-
-    lapi_credentials_path=$(conf_get '.api.client.credentials_path')
-
-    if istrue "$USE_TLS"; then
-        install -m 0600 /dev/null "$lapi_credentials_path"
-        conf_set '
-            .url = strenv(LOCAL_API_URL) |
-            .ca_cert_path = strenv(CACERT_FILE) |
-            .key_path = strenv(KEY_FILE) |
-            .cert_path = strenv(CERT_FILE)
-        ' "$lapi_credentials_path"
-    elif [ "$AGENT_USERNAME" != "" ]; then
-        install -m 0600 /dev/null "$lapi_credentials_path"
-        conf_set '
-            .url = strenv(LOCAL_API_URL) |
-            .login = strenv(AGENT_USERNAME) |
-            .password = strenv(AGENT_PASSWORD)
-        ' "$lapi_credentials_path"
-    fi
-fi
-
 if isfalse "$DISABLE_LOCAL_API"; then
-    echo "Check if lapi needs to automatically register an agent"
+    if isfalse "$USE_TLS" || [ "$CLIENT_CERT_FILE" = "" ]; then
+        echo "Regenerate local agent credentials"
+        cscli machines add "$CUSTOM_HOSTNAME" --auto
+    fi
 
-    # pre-registration is not needed with TLS
+    echo "Check if lapi needs to register an additional agent"
-    if isfalse "$USE_TLS" && [ "$AGENT_USERNAME" != "" ] && [ "$AGENT_PASSWORD" != "" ] ; then
+    # pre-registration is not needed with TLS authentication, but we can have TLS transport with user/pw
-        # shellcheck disable=SC2086
+    if [ "$AGENT_USERNAME" != "" ] && [ "$AGENT_PASSWORD" != "" ] ; then
-        cscli machines add "$AGENT_USERNAME" --password "$AGENT_PASSWORD" --url "$LOCAL_API_URL"
+        # re-register because pw may have been changed
+        cscli machines add "$AGENT_USERNAME" --password "$AGENT_PASSWORD" -f /dev/null --force
         echo "Agent registered to lapi"
     fi
 fi
 
+# ----------------
+
+lapi_credentials_path=$(conf_get '.api.client.credentials_path')
+
+conf_set_if "$LOCAL_API_URL" '.url = strenv(LOCAL_API_URL)' "$lapi_credentials_path"
+
+if istrue "$DISABLE_LOCAL_API"; then
+    # we only use the envvars that are actually defined
+    # in case of persistent configuration
+    conf_set_if "$AGENT_USERNAME" '.login = strenv(AGENT_USERNAME)' "$lapi_credentials_path"
+    conf_set_if "$AGENT_PASSWORD" '.password = strenv(AGENT_PASSWORD)' "$lapi_credentials_path"
+fi
+
+conf_set_if "$INSECURE_SKIP_VERIFY" '.api.client.insecure_skip_verify = env(INSECURE_SKIP_VERIFY)'
+
+# agent-only containers still require USE_TLS
+if istrue "$USE_TLS"; then
+    # shellcheck disable=SC2153
+    conf_set_if "$CACERT_FILE" '.ca_cert_path = strenv(CACERT_FILE)' "$lapi_credentials_path"
+    conf_set_if "$CLIENT_KEY_FILE" '.key_path = strenv(CLIENT_KEY_FILE)' "$lapi_credentials_path"
+    conf_set_if "$CLIENT_CERT_FILE" '.cert_path = strenv(CLIENT_CERT_FILE)' "$lapi_credentials_path"
+else
+    conf_set '
+        del(.ca_cert_path) |
+        del(.key_path) |
+        del(.cert_path)
+    ' "$lapi_credentials_path"
+fi
+
+if istrue "$DISABLE_ONLINE_API"; then
+    conf_set 'del(.api.server.online_client)'
+fi
+
 # registration to online API for signal push
-if isfalse "$DISABLE_ONLINE_API" && [ "$CONFIG_FILE" == "/etc/crowdsec/config.yaml" ] ; then
+if isfalse "$DISABLE_ONLINE_API" ; then
+    CONFIG_DIR=$(conf_get '.config_paths.config_dir')
     config_exists=$(conf_get '.api.server.online_client | has("credentials_path")')
     if isfalse "$config_exists"; then
-        conf_set '.api.server.online_client = {"credentials_path": "/etc/crowdsec/online_api_credentials.yaml"}'
+        export CONFIG_DIR
-        cscli capi register > /etc/crowdsec/online_api_credentials.yaml
+        conf_set '.api.server.online_client = {"credentials_path": strenv(CONFIG_DIR) + "/online_api_credentials.yaml"}'
+        cscli capi register > "$CONFIG_DIR/online_api_credentials.yaml"
         echo "Registration to online API done"
     fi
 fi
@@ -158,84 +241,75 @@ if [ "$GID" != "" ]; then
     fi
 fi
 
+# XXX only with LAPI
 if istrue "$USE_TLS"; then
     agents_allowed_yaml=$(csv2yaml "$AGENTS_ALLOWED_OU") \
     bouncers_allowed_yaml=$(csv2yaml "$BOUNCERS_ALLOWED_OU") \
-    conf_set '
+    conf_set_if "$CACERT_FILE" '.api.server.tls.ca_cert_path = strenv(CACERT_FILE)'
-        .api.server.tls.ca_cert_path = strenv(CACERT_FILE) |
+    conf_set_if "$LAPI_CERT_FILE" '.api.server.tls.cert_file = strenv(LAPI_CERT_FILE)'
-        .api.server.tls.cert_file = strenv(CERT_FILE) |
+    conf_set_if "$LAPI_KEY_FILE" '.api.server.tls.key_file = strenv(LAPI_KEY_FILE)'
-        .api.server.tls.key_file = strenv(KEY_FILE) |
+    conf_set_if "$BOUNCERS_ALLOWED_OU" '.api.server.tls.bouncers_allowed_ou = env(bouncers_allowed_yaml)'
-        .api.server.tls.bouncers_allowed_ou = env(bouncers_allowed_yaml) |
+    conf_set_if "$AGENTS_ALLOWED_OU" '.api.server.tls.agents_allowed_ou = env(agents_allowed_yaml)'
-        .api.server.tls.agents_allowed_ou = env(agents_allowed_yaml) |
+else
-        ... comments=""
+    conf_set 'del(.api.server.tls)'
-        '
 fi
 
-conf_set ".config_paths.plugin_dir = strenv(PLUGIN_DIR)"
+conf_set_if "$PLUGIN_DIR" '.config_paths.plugin_dir = strenv(PLUGIN_DIR)'
 
 ## Install collections, parsers, scenarios & postoverflows
 cscli hub update
-cscli collections upgrade crowdsecurity/linux || true
+
-cscli parsers upgrade crowdsecurity/whitelists || true
+cscli_if_clean collections upgrade crowdsecurity/linux
-cscli parsers install crowdsecurity/docker-logs || true
+cscli_if_clean parsers upgrade crowdsecurity/whitelists
+cscli_if_clean parsers install crowdsecurity/docker-logs
 
 if [ "$COLLECTIONS" != "" ]; then
     # shellcheck disable=SC2086
-    cscli collections install $COLLECTIONS
+    cscli_if_clean collections install "$COLLECTIONS"
 fi
 
 if [ "$PARSERS" != "" ]; then
     # shellcheck disable=SC2086
-    cscli parsers install $PARSERS
+    cscli_if_clean parsers install "$PARSERS"
 fi
 
 if [ "$SCENARIOS" != "" ]; then
     # shellcheck disable=SC2086
-    cscli scenarios install $SCENARIOS
+    cscli_if_clean scenarios install "$SCENARIOS"
 fi
 
 if [ "$POSTOVERFLOWS" != "" ]; then
     # shellcheck disable=SC2086
-    cscli postoverflows install $POSTOVERFLOWS
+    cscli_if_clean postoverflows install "$POSTOVERFLOWS"
 fi
 
 ## Remove collections, parsers, scenarios & postoverflows
 if [ "$DISABLE_COLLECTIONS" != "" ]; then
     # shellcheck disable=SC2086
-    cscli collections remove $DISABLE_COLLECTIONS
+    cscli_if_clean collections remove "$DISABLE_COLLECTIONS"
 fi
 
 if [ "$DISABLE_PARSERS" != "" ]; then
     # shellcheck disable=SC2086
-    cscli parsers remove $DISABLE_PARSERS
+    cscli_if_clean parsers remove "$DISABLE_PARSERS"
 fi
 
 if [ "$DISABLE_SCENARIOS" != "" ]; then
     # shellcheck disable=SC2086
-    cscli scenarios remove $DISABLE_SCENARIOS
+    cscli_if_clean scenarios remove "$DISABLE_SCENARIOS"
 fi
 
 if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
     # shellcheck disable=SC2086
-    cscli postoverflows remove $DISABLE_POSTOVERFLOWS
+    cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS"
 fi
 
-register_bouncer() {
-  if ! cscli bouncers list -o json | sed '/^ *"name"/!d;s/^ *"name": "\(.*\)",/\1/' | grep -q "^${NAME}$"; then
-      if cscli bouncers add "${NAME}" -k "${KEY}" > /dev/null; then
-          echo "Registered bouncer for ${NAME}"
-      else
-          echo "Failed to register bouncer for ${NAME}"
-      fi
-  fi
-}
-
 ## Register bouncers via env
 for BOUNCER in $(compgen -A variable | grep -i BOUNCER_KEY); do
     KEY=$(printf '%s' "${!BOUNCER}")
-    NAME=$(printf '%s' "$BOUNCER" | cut -d_  -f2-)
+    NAME=$(printf '%s' "$BOUNCER" | cut -d_  -f3-)
     if [[ -n $KEY ]] && [[ -n $NAME ]]; then
-        register_bouncer
+        register_bouncer "$NAME" "$KEY"
     fi
 done
 
@@ -245,7 +319,7 @@ for BOUNCER in /run/secrets/@(bouncer_key|BOUNCER_KEY)* ; do
     KEY=$(cat "${BOUNCER}")
     NAME=$(echo "${BOUNCER}" | awk -F "/" '{printf $NF}' | cut -d_  -f2-)
     if [[ -n $KEY ]] && [[ -n $NAME ]]; then    
-        register_bouncer
+        register_bouncer "$NAME" "$KEY"
     fi
 done
 shopt -u nullglob extglob
@@ -287,7 +361,7 @@ if istrue "$LEVEL_INFO"; then
     ARGS="$ARGS -info"
 fi
 
-conf_set '.prometheus.listen_port=env(METRICS_PORT)'
+conf_set_if "$METRICS_PORT" '.prometheus.listen_port=env(METRICS_PORT)'
 
 # shellcheck disable=SC2086
 exec crowdsec $ARGS

pkg/acquisition/acquisition.go

@@ -25,6 +25,20 @@ import (
 	tomb "gopkg.in/tomb.v2"
 )
 
+type DataSourceUnavailableError struct {
+	Name string
+	Err error
+}
+
+func (e *DataSourceUnavailableError) Error() string {
+	return fmt.Sprintf("datasource '%s' is not available: %v", e.Name, e.Err)
+}
+
+func (e *DataSourceUnavailableError) Unwrap() error {
+	return e.Err
+}
+
+
 // The interface each datasource must implement
 type DataSource interface {
 	GetMetrics() []prometheus.Collector                         // Returns pointers to metrics that are managed by the module
@@ -86,8 +100,11 @@ func GetDataSourceIface(dataSourceType string) DataSource {
 	return nil
 }
 
+// DataSourceConfigure creates and returns a DataSource object from a configuration,
+// if the configuration is not valid it returns an error.
+// If the datasource can't be run (eg. journalctl not available), it still returns an error which
+// can be checked for the appropriate action.
 func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataSource, error) {
-
 	//we dump it back to []byte, because we want to decode the yaml blob twice :
 	//once to DataSourceCommonCfg, and then later to the dedicated type of the datasource
 	yamlConfig, err := yaml.Marshal(commonConfig)
@@ -112,7 +129,7 @@ func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataS
 		subLogger := clog.WithFields(customLog)
 		/* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */
 		if err := dataSrc.CanRun(); err != nil {
-			return nil, errors.Wrapf(err, "datasource %s cannot be run", commonConfig.Source)
+			return nil, &DataSourceUnavailableError{Name: commonConfig.Source, Err: err}
 		}
 		/* configure the actual datasource */
 		if err := dataSrc.Configure(yamlConfig, subLogger); err != nil {
@@ -179,10 +196,11 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
 		}
 		dec := yaml.NewDecoder(yamlFile)
 		dec.SetStrict(true)
+		idx := -1
 		for {
 			var sub configuration.DataSourceCommonCfg
-			var idx int
 			err = dec.Decode(&sub)
+			idx += 1
 			if err != nil {
 				if ! errors.Is(err, io.EOF) {
 					return nil, errors.Wrapf(err, "failed to yaml decode %s", acquisFile)
@@ -199,7 +217,6 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
 			if len(sub.Labels) == 0 {
 				if sub.Source == "" {
 					log.Debugf("skipping empty item in %s", acquisFile)
-					idx += 1
 					continue
 				}
 				return nil, fmt.Errorf("missing labels in %s (position: %d)", acquisFile, idx)
@@ -212,10 +229,14 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
 			}
 			src, err := DataSourceConfigure(sub)
 			if err != nil {
+				var dserr *DataSourceUnavailableError
+				if errors.As(err, &dserr) {
+					log.Error(err)
+					continue
+				}
 				return nil, errors.Wrapf(err, "while configuring datasource of type %s from %s (position: %d)", sub.Source, acquisFile, idx)
 			}
 			sources = append(sources, *src)
-			idx += 1
 		}
 	}
 	return sources, nil

pkg/acquisition/acquisition_test.go

@@ -171,7 +171,7 @@ log_level: debug
 source: mock_cant_run
 wowo: ajsajasjas
 `,
-			ExpectedError: "datasource mock_cant_run cannot be run: can't run bro",
+			ExpectedError: "datasource 'mock_cant_run' is not available: can't run bro",
 		},
 	}
 

pkg/acquisition/modules/syslog/internal/parser/rfc3164/parse_test.go

@@ -54,7 +54,7 @@ func TestTimestamp(t *testing.T) {
 		currentYear bool
 	}{
 		{"May 20 09:33:54", "0000-05-20T09:33:54Z", "", false},
-		{"May 20 09:33:54", "2022-05-20T09:33:54Z", "", true},
+		{"May 20 09:33:54", "2023-05-20T09:33:54Z", "", true},
 		{"May 20 09:33:54 2022", "2022-05-20T09:33:54Z", "", false},
 		{"May  1 09:33:54 2022", "2022-05-01T09:33:54Z", "", false},
 		{"May 01 09:33:54 2021", "2021-05-01T09:33:54Z", "", true},
@@ -257,7 +257,7 @@ func TestParse(t *testing.T) {
 		},
 		{
 			"<12>May 20 09:33:54 UDMPRO,a2edd0c6ae48,udm-1.10.0.3686 kernel: foo", expected{
-				Timestamp: time.Date(2022, time.May, 20, 9, 33, 54, 0, time.UTC),
+				Timestamp: time.Date(2023, time.May, 20, 9, 33, 54, 0, time.UTC),
 				Hostname:  "UDMPRO,a2edd0c6ae48,udm-1.10.0.3686",
 				Tag:       "kernel",
 				PID:       "",

pkg/apiclient/client.go

@@ -53,8 +53,8 @@ func NewClient(config *Config) (*ApiClient, error) {
 		UpdateScenario: config.UpdateScenario,
 	}
 	tlsconfig := tls.Config{InsecureSkipVerify: InsecureSkipVerify}
-	if Cert != nil {
 	tlsconfig.RootCAs = CaCertPool
+	if Cert != nil {
 		tlsconfig.Certificates = []tls.Certificate{*Cert}
 	}
 	http.DefaultTransport.(*http.Transport).TLSClientConfig = &tlsconfig
@@ -75,8 +75,8 @@ func NewDefaultClient(URL *url.URL, prefix string, userAgent string, client *htt
 		client = &http.Client{}
 		if ht, ok := http.DefaultTransport.(*http.Transport); ok {
 			tlsconfig := tls.Config{InsecureSkipVerify: InsecureSkipVerify}
-			if Cert != nil {
 			tlsconfig.RootCAs = CaCertPool
+			if Cert != nil {
 				tlsconfig.Certificates = []tls.Certificate{*Cert}
 			}
 			ht.TLSClientConfig = &tlsconfig

pkg/csconfig/api.go

@@ -81,7 +81,7 @@ func (l *LocalApiClientCfg) Load() error {
 		}
 	}
 
-	if l.Credentials.Login != "" && (l.Credentials.CACertPath != "" || l.Credentials.CertPath != "" || l.Credentials.KeyPath != "") {
+	if l.Credentials.Login != "" && (l.Credentials.CertPath != "" || l.Credentials.KeyPath != "") {
 		return fmt.Errorf("user/password authentication and TLS authentication are mutually exclusive")
 	}
 
@@ -91,12 +91,7 @@ func (l *LocalApiClientCfg) Load() error {
 		apiclient.InsecureSkipVerify = *l.InsecureSkipVerify
 	}
 
-	if l.Credentials.CACertPath != "" && l.Credentials.CertPath != "" && l.Credentials.KeyPath != "" {
+	if l.Credentials.CACertPath != ""  {
-		cert, err := tls.LoadX509KeyPair(l.Credentials.CertPath, l.Credentials.KeyPath)
-		if err != nil {
-			return errors.Wrapf(err, "failed to load api client certificate")
-		}
-
 		caCert, err := os.ReadFile(l.Credentials.CACertPath)
 		if err != nil {
 			return errors.Wrapf(err, "failed to load cacert")
@@ -104,10 +99,18 @@ func (l *LocalApiClientCfg) Load() error {
 
 		caCertPool := x509.NewCertPool()
 		caCertPool.AppendCertsFromPEM(caCert)
-
-		apiclient.Cert = &cert
 		apiclient.CaCertPool = caCertPool
 	}
+
+	if l.Credentials.CertPath != "" && l.Credentials.KeyPath != "" {
+		cert, err := tls.LoadX509KeyPair(l.Credentials.CertPath, l.Credentials.KeyPath)
+		if err != nil {
+			return errors.Wrapf(err, "failed to load api client certificate")
+		}
+
+		apiclient.Cert = &cert
+	}
+
 	return nil
 }
 

tests/bats/01_crowdsec.bats

@@ -148,9 +148,10 @@ teardown() {
     rm -f "$ACQUIS_DIR"
 
     config_set '.common.log_media="stdout"'
-    run -124 --separate-stderr timeout 2s "${CROWDSEC}"
+    run -1 --separate-stderr timeout 2s "${CROWDSEC}"
     # check warning
-    assert_stderr_line --partial "no acquisition file found"
+    assert_stderr --partial "no acquisition file found"
+    assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
 }
 
 @test "crowdsec (error if acquisition_path and acquisition_dir are not defined)" {
@@ -163,19 +164,55 @@ teardown() {
     config_set '.crowdsec_service.acquisition_dir=""'
 
     config_set '.common.log_media="stdout"'
-    run -124 --separate-stderr timeout 2s "${CROWDSEC}"
+    run -1 --separate-stderr timeout 2s "${CROWDSEC}"
     # check warning
-    assert_stderr_line --partial "no acquisition_path or acquisition_dir specified"
+    assert_stderr --partial "no acquisition_path or acquisition_dir specified"
+    assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
 }
 
 @test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" {
     ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
-    rm -f "$ACQUIS_YAML"
     config_set '.crowdsec_service.acquisition_path=""'
 
     ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir')
     mkdir -p "$ACQUIS_DIR"
-    touch "$ACQUIS_DIR"/foo.yaml
+    mv "$ACQUIS_YAML" "$ACQUIS_DIR"/foo.yaml
 
     run -124 --separate-stderr timeout 2s "${CROWDSEC}"
+
+    # now, if foo.yaml is empty instead, there won't be valid datasources.
+
+    cat /dev/null >"$ACQUIS_DIR"/foo.yaml
+
+    run --separate-stderr -1 timeout 2s "${CROWDSEC}"
+    assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
+}
+
+@test "crowdsec (disabled datasources)" {
+    config_set '.common.log_media="stdout"'
+
+    # a datasource cannot run - missing journalctl command
+
+    ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir')
+    mkdir -p "$ACQUIS_DIR"
+    cat >"$ACQUIS_DIR"/foo.yaml <<-EOT
+	source: journalctl
+	journalctl_filter:
+	- "_SYSTEMD_UNIT=ssh.service"
+	labels:
+	  type: syslog
+	EOT
+
+    run --separate-stderr -124 timeout 2s env PATH='' "${CROWDSEC}"
+    #shellcheck disable=SC2016
+    assert_stderr --partial 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in $PATH'
+
+    # if all datasources are disabled, crowdsec should exit
+
+    ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
+    rm -f "$ACQUIS_YAML"
+    config_set '.crowdsec_service.acquisition_path=""'
+
+    run --separate-stderr -1 timeout 2s env PATH='' "${CROWDSEC}"
+    assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
  }

tests/bin/assert-crowdsec-not-running

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 is_crowdsec_running() {
-    PIDS=$(pgrep -x 'crowdsec|crowdsec.test|crowdsec.cover')
+    PIDS=$(pgrep -x 'crowdsec|crowdsec.test|crowdsec.cover' 2>/dev/null)
 }
 
 # The process can be slow, especially on CI and during test coverage.