non-fatal error if some datasource can't be run (i.e. journalctl but systemd is missing) (#2310 )

This on the other hand, gives a new fatal error when there are no valid datasources. In the previous version, crowdsec kept running with just a warning if no acquisition yaml or dir were specified.
strip v prefix in tag name in azure-pipeline (#2048 ) (#2049 )
2023-06-27 10:46:25 +02:00 · 2023-02-09 14:27:11 +01:00 · 2023-02-09 13:15:36 +01:00 · 2023-02-09 11:01:11 +01:00 · 2023-02-09 10:23:07 +01:00 · 2023-01-15 09:04:50 +01:00
23 changed files with 381 additions and 381 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -1,3 +1,4 @@
 # We include .git in the build context because excluding it would break the
 # "make release" target, which uses git to retrieve the build version and tag.
 #.git
+/tests/ansible
--- a/.github/workflows/ci-windows-build-msi.yml
+++ b/.github/workflows/ci-windows-build-msi.yml
@ -1,17 +1,11 @@
 name: build-msi (windows)

 on:
-  pull_request:
-    branches:
-      - master
-      - releases/**
-    paths-ignore:
-      - 'docs/**'
-      - 'mkdocs.yml'
-      - 'README.md'
+  release:
+    types:
+      - prereleased

 jobs:
-
  build:
    name: Build
    runs-on: windows-2019
@ -22,14 +16,9 @@ jobs:
        go-version: 1.19
      id: go
    - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
-    - id: get_latest_release
-      uses: pozetroninc/github-action-get-latest-release@master
+      uses: actions/checkout@v3
      with:
-        repository: crowdsecurity/crowdsec
-        excludes: draft
-    - id: set_release_in_env
-      run: echo "BUILD_VERSION=${{ steps.get_latest_release.outputs.release }}" >> $env:GITHUB_ENV
+        fetch-depth: 0
    - name: Build
      run: make windows_installer
    - name: Upload MSI
--- a/.github/workflows/dispatch_ci_hub.yaml
+++ b/.github/workflows/dispatch_ci_hub.yaml
@ -1,22 +0,0 @@
-name: Dispatch to hub when creating pre-release
-
-on:
-  push:
-    # branches to consider in the event; optional, defaults to all
-    branches:
-      - master
-      - releases/**
-
-jobs:
-  dispatch:
-    name: dispatch to hub-tests
-    runs-on: ubuntu-latest
-    steps:
-      - name: Repository Dispatch
-        uses: peter-evans/repository-dispatch@v1
-        if: ${{ github.repository_owner == 'crowdsecurity' }}
-        with:
-          token: ${{ secrets.DISPATCH_TOKEN }}
-          event-type: trigger_ci_hub
-          repository: crowdsecurity/hub
-          client-payload: '{"version": "${{ steps.keydb.outputs.release }}"}'
--- a/.github/workflows/dispatch_create_branch_hub.yaml
+++ b/.github/workflows/dispatch_create_branch_hub.yaml
@ -1,24 +0,0 @@
-name: Dispatch to hub when creating pre-release
-
-on:
-  release:
-    types: prereleased
-
-jobs:
-  dispatch:
-    name: dispatch to hub-tests
-    runs-on: ubuntu-latest
-    steps:
-      - id: keydb
-        uses: pozetroninc/github-action-get-latest-release@master
-        with:
-          owner: crowdsecurity
-          repo: crowdsec
-          excludes: prerelease, draft
-      - name: Repository Dispatch
-        uses: peter-evans/repository-dispatch@v1
-        with:
-          token: ${{ secrets.DISPATCH_TOKEN }}
-          event-type: create_branch
-          repository: crowdsecurity/hub
-          client-payload: '{"version": "${{ steps.keydb.outputs.release }}"}'
--- a/.github/workflows/dispatch_delete_branch_hub.yaml
+++ b/.github/workflows/dispatch_delete_branch_hub.yaml
@ -1,24 +0,0 @@
-name: Dispatch to hub when deleting pre-release
-
-on:
-  release:
-    types: deleted
-
-jobs:
-  dispatch:
-    name: dispatch to hub-tests
-    runs-on: ubuntu-latest
-    steps:
-      - id: keydb
-        uses: pozetroninc/github-action-get-latest-release@master
-        with:
-          owner: crowdsecurity
-          repo: crowdsec
-          excludes: prerelease, draft
-      - name: Repository Dispatch
-        uses: peter-evans/repository-dispatch@v1
-        with:
-          token: ${{ secrets.DISPATCH_TOKEN }}
-          event-type: delete_branch
-          repository: crowdsecurity/hub
-          client-payload: '{"version": "${{ steps.keydb.outputs.release }}"}'
--- a/.github/workflows/release_publish-package.yml
+++ b/.github/workflows/release_publish-package.yml
@ -3,46 +3,51 @@ name: build

 on:
  release:
-    types: prereleased
+    types:
+      - prereleased

 jobs:
  build:
    name: Build and upload binary package
    runs-on: ubuntu-latest
    steps:
-    - name: Set up Go 1.19
-      uses: actions/setup-go@v3
-      with:
-        go-version: 1.19
-      id: go
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v3
-    - name: Build the binaries
-      run: make release
-    - name: Upload to release
-      uses: JasonEtco/upload-to-release@master
-      with:
-        args: crowdsec-release.tgz application/x-gzip
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+        id: go
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Build the binaries
+        run: make release
+      - name: Upload to release
+        uses: JasonEtco/upload-to-release@master
+        with:
+          args: crowdsec-release.tgz application/x-gzip
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  build_static:
    name: Build and upload binary package
    runs-on: ubuntu-latest
    steps:
-    - name: Set up Go 1.19
-      uses: actions/setup-go@v3
-      with:
-        go-version: 1.19
-      id: go
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v3
-    - name: Build the binaries
-      run: |
-        make release BUILD_STATIC=yes
-        mv crowdsec-release.tgz crowdsec-release-static.tgz
-    - name: Upload to release
-      uses: JasonEtco/upload-to-release@master
-      with:
-        args: crowdsec-release-static.tgz application/x-gzip
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+        id: go
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Build the binaries
+        run: |
+          make release BUILD_STATIC=yes
+          mv crowdsec-release.tgz crowdsec-release-static.tgz
+      - name: Upload to release
+        uses: JasonEtco/upload-to-release@master
+        with:
+          args: crowdsec-release-static.tgz application/x-gzip
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
--- a/.github/workflows/release_publish_docker-image.yml
+++ b/.github/workflows/release_publish_docker-image.yml
@ -32,7 +32,7 @@ jobs:
          TAGS_SLIM="${DOCKER_IMAGE}:${VERSION}-slim"
          if [[ ${{ github.event.action }} == released ]]; then
            TAGS=$TAGS,${DOCKER_IMAGE}:latest,${GHCR_IMAGE}:latest
-            TAGS_SLIM=$TAGS,${DOCKER_IMAGE}:slim
+            TAGS_SLIM=$TAGS_SLIM,${DOCKER_IMAGE}:slim
          fi
          echo ::set-output name=version::${VERSION}
          echo ::set-output name=tags::${TAGS}
--- a/67
+++ b/67
@ -1,3 +1,4 @@
+# vim: set ft=dockerfile:
 ARG BUILD_ENV=full
 ARG GOVERSION=1.19

@ -9,82 +10,30 @@ COPY . .

 # wizard.sh requires GNU coreutils
 RUN apk add --no-cache git gcc libc-dev make bash gettext binutils-gold coreutils && \
+    echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
    SYSTEM="docker" make clean release && \
    cd crowdsec-v* && \
    ./wizard.sh --docker-mode && \
-    cd - && \
+    cd - >/dev/null && \
    cscli hub update && \
    cscli collections install crowdsecurity/linux && \
-    cscli parsers install crowdsecurity/whitelists
+    cscli parsers install crowdsecurity/whitelists && \
+    go install github.com/mikefarah/yq/v4@v4.30.6

 FROM alpine:latest as build-slim

-RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata yq bash && \
+RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata bash && \
    mkdir -p /staging/etc/crowdsec && \
    mkdir -p /staging/var/lib/crowdsec && \
    mkdir -p /var/lib/crowdsec/data

+COPY --from=build /go/bin/yq /usr/local/bin/yq
 COPY --from=build /etc/crowdsec /staging/etc/crowdsec
 COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
 COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
 COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
 COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
-
-ENV CONFIG_FILE=/etc/crowdsec/config.yaml
-ENV LOCAL_API_URL=http://0.0.0.0:8080/
-ENV CUSTOM_HOSTNAME=localhost
-ENV PLUGIN_DIR=/usr/local/lib/crowdsec/plugins/
-ENV DISABLE_AGENT=false
-ENV DISABLE_LOCAL_API=false
-ENV DISABLE_ONLINE_API=false
-ENV DSN=
-ENV TYPE=
-ENV TEST_MODE=false
-ENV USE_WAL=false
-
-# register to app.crowdsec.net
-
-ENV ENROLL_INSTANCE_NAME=
-ENV ENROLL_KEY=
-ENV ENROLL_TAGS=
-
-# log verbosity
-
-ENV LEVEL_TRACE=false
-ENV LEVEL_DEBUG=false
-ENV LEVEL_INFO=true
-
-# TLS setup ----------------------------------- #
-
-ENV AGENT_USERNAME=
-ENV AGENT_PASSWORD=
-
-# TLS setup ----------------------------------- #
-
-ENV USE_TLS=false
-ENV CA_CERT_PATH=
-ENV CERT_FILE=/etc/ssl/cert.pem
-ENV KEY_FILE=/etc/ssl/key.pem
-# comma-separated list of allowed OU values for TLS bouncer certificates
-ENV BOUNCERS_ALLOWED_OU=bouncer-ou
-# comma-separated list of allowed OU values for TLS agent certificates
-ENV AGENTS_ALLOWED_OU=agent-ou
-
-# Install the following hub items --------------#
-
-ENV COLLECTIONS=
-ENV PARSERS=
-ENV SCENARIOS=
-ENV POSTOVERFLOWS=
-
-# Uninstall the following hub items ------------#
-
-ENV DISABLE_COLLECTIONS=
-ENV DISABLE_PARSERS=
-ENV DISABLE_SCENARIOS=
-ENV DISABLE_POSTOVERFLOWS=
-
-ENV METRICS_PORT=6060
+RUN yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml

 ENTRYPOINT /bin/bash docker_start.sh

--- a/Dockerfile.debian
+++ b/Dockerfile.debian
@ -8,20 +8,27 @@ WORKDIR /go/src/crowdsec

 COPY . .

+ENV DEBIAN_FRONTEND=noninteractive
+ENV DEBCONF_NOWARNINGS="yes"
+
 # wizard.sh requires GNU coreutils
 RUN apt-get update && \
-    apt-get install -y git gcc libc-dev make bash gettext binutils-gold coreutils tzdata && \
-    SYSTEM="docker" make release && \
+    apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata && \
+    echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
+    SYSTEM="docker" make clean release && \
    cd crowdsec-v* && \
    ./wizard.sh --docker-mode && \
-    cd - && \
+    cd - >/dev/null && \
    cscli hub update && \
    cscli collections install crowdsecurity/linux && \
    cscli parsers install crowdsecurity/whitelists && \
-    go install github.com/mikefarah/yq/v4@latest
+    go install github.com/mikefarah/yq/v4@v4.30.6

 FROM debian:bullseye-slim as build-slim

+ENV DEBIAN_FRONTEND=noninteractive
+ENV DEBCONF_NOWARNINGS="yes"
+
 RUN apt-get update && \
    apt-get install -y -q --install-recommends --no-install-suggests \
    procps \
@ -40,63 +47,9 @@ COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
 COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
 COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
 COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
-RUN yq eval -i ".plugin_config.group = \"nogroup\"" /staging/etc/crowdsec/config.yaml
+RUN yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml && \
+    yq eval -i ".plugin_config.group = \"nogroup\"" /staging/etc/crowdsec/config.yaml

-ENV CONFIG_FILE=/etc/crowdsec/config.yaml
-ENV LOCAL_API_URL=http://0.0.0.0:8080/
-ENV CUSTOM_HOSTNAME=localhost
-ENV PLUGIN_DIR=/usr/local/lib/crowdsec/plugins/
-ENV DISABLE_AGENT=false
-ENV DISABLE_LOCAL_API=false
-ENV DISABLE_ONLINE_API=false
-ENV DSN=
-ENV TYPE=
-ENV TEST_MODE=false
-ENV USE_WAL=false
-
-# register to app.crowdsec.net
-
-ENV ENROLL_INSTANCE_NAME=
-ENV ENROLL_KEY=
-ENV ENROLL_TAGS=
-
-# log verbosity
-
-ENV LEVEL_TRACE=false
-ENV LEVEL_DEBUG=false
-ENV LEVEL_INFO=true
-
-# TLS setup ----------------------------------- #
-
-ENV AGENT_USERNAME=
-ENV AGENT_PASSWORD=
-
-# TLS setup ----------------------------------- #
-
-ENV USE_TLS=false
-ENV CA_CERT_PATH=
-ENV CERT_FILE=/etc/ssl/cert.pem
-ENV KEY_FILE=/etc/ssl/key.pem
-# comma-separated list of allowed OU values for TLS bouncer certificates
-ENV BOUNCERS_ALLOWED_OU=bouncer-ou
-# comma-separated list of allowed OU values for TLS agent certificates
-ENV AGENTS_ALLOWED_OU=agent-ou
-
-# Install the following hub items --------------#
-
-ENV COLLECTIONS=
-ENV PARSERS=
-ENV SCENARIOS=
-ENV POSTOVERFLOWS=
-
-# Uninstall the following hub items ------------#
-
-ENV DISABLE_COLLECTIONS=
-ENV DISABLE_PARSERS=
-ENV DISABLE_SCENARIOS=
-ENV DISABLE_POSTOVERFLOWS=
-
-ENV METRICS_PORT=6060

 ENTRYPOINT /bin/bash docker_start.sh

--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@ -56,25 +56,26 @@ stages:
              --input "**/*.exe" --config (Join-Path -Path $(Agent.TempDirectory) -ChildPath "appsettings.json") `
              --user $(CodeSigningUser) --secret '$(CodeSigningPassword)'
            displayName: "Sign Crowdsec binaries + plugins"
-
          - pwsh: |
-              $build_version=(git describe --tags (git rev-list --tags --max-count=1)).Substring(1)
+              $build_version=$env:BUILD_SOURCEBRANCHNAME
+              if ($build_version.StartsWith("v"))
+              {
+                $build_version = $build_version.Substring(1)
+              }
              if ($build_version.Contains("-")) 
              {
                $build_version = $build_version.Substring(0, $build_version.IndexOf("-"))
              }
-              .\make_installer.ps1 -version $build_version
              Write-Host  "##vso[task.setvariable variable=BuildVersion;isOutput=true]$build_version"
+            displayName: GetCrowdsecVersion
+            name: GetCrowdsecVersion
+          - pwsh: |
+              .\make_installer.ps1 -version '$(GetCrowdsecVersion.BuildVersion)'
            displayName: "Build Crowdsec MSI"
            name: BuildMSI

          - pwsh: |
-              $build_version=(git describe --tags (git rev-list --tags --max-count=1)).Substring(1)
-              if ($build_version.Contains("-")) 
-              {
-                $build_version = $build_version.Substring(0, $build_version.IndexOf("-"))
-              }
-              .\make_chocolatey.ps1 -version $build_version
+              .\make_chocolatey.ps1 -version '$(GetCrowdsecVersion.BuildVersion)'
            displayName: "Build Chocolatey nupkg"

          - pwsh: |
@ -85,14 +86,14 @@ stages:

          - task: PublishBuildArtifacts@1
            inputs:
-              PathtoPublish: '$(Build.Repository.LocalPath)\\crowdsec_$(BuildMSI.BuildVersion).msi'
+              PathtoPublish: '$(Build.Repository.LocalPath)\\crowdsec_$(GetCrowdsecVersion.BuildVersion).msi'
              ArtifactName: 'crowdsec.msi'
              publishLocation: 'Container'
            displayName: "Upload MSI artifact"
          
          - task: PublishBuildArtifacts@1
            inputs:
-              PathtoPublish: '$(Build.Repository.LocalPath)\\windows\\Chocolatey\\crowdsec\\crowdsec.$(BuildMSI.BuildVersion).nupkg'
+              PathtoPublish: '$(Build.Repository.LocalPath)\\windows\\Chocolatey\\crowdsec\\crowdsec.$(GetCrowdsecVersion.BuildVersion).nupkg'
              ArtifactName: 'crowdsec.nupkg'
              publishLocation: 'Container'
            displayName: "Upload nupkg artifact"
--- a/cmd/crowdsec-cli/main.go
+++ b/cmd/crowdsec-cli/main.go
@ -52,7 +52,7 @@ func initConfig() {
 	} else if err_lvl {
 		log.SetLevel(log.ErrorLevel)
 	}
-	logFormatter := &log.TextFormatter{TimestampFormat: "02-01-2006 03:04:05 PM", FullTimestamp: true}
+	logFormatter := &log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}
 	log.SetFormatter(logFormatter)

 	if !inSlice(os.Args[1], NoNeedConfig) {
--- a/cmd/crowdsec/crowdsec.go
+++ b/cmd/crowdsec/crowdsec.go
@ -23,22 +23,22 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
 	var err error

 	// Populate cwhub package tools
-	if err := cwhub.GetHubIdx(cConfig.Hub); err != nil {
-		return &parser.Parsers{}, fmt.Errorf("Failed to load hub index : %s", err)
+	if err = cwhub.GetHubIdx(cConfig.Hub); err != nil {
+		return &parser.Parsers{}, fmt.Errorf("while loading hub index : %s", err)
 	}

 	// Start loading configs
 	csParsers := newParsers()
 	if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
-		return &parser.Parsers{}, fmt.Errorf("Failed to load parsers: %s", err)
+		return nil, fmt.Errorf("while loading parsers: %s", err)
 	}

 	if err := LoadBuckets(cConfig); err != nil {
-		return &parser.Parsers{}, fmt.Errorf("Failed to load scenarios: %s", err)
+		return nil, fmt.Errorf("while loading scenarios: %s", err)
 	}

 	if err := LoadAcquisition(cConfig); err != nil {
-		return &parser.Parsers{}, fmt.Errorf("Error while loading acquisition config : %s", err)
+		return nil, fmt.Errorf("while loading acquisition config: %s", err)
 	}
 	return csParsers, nil
 }
--- a/cmd/crowdsec/main.go
+++ b/cmd/crowdsec/main.go
@ -157,6 +157,10 @@ func LoadAcquisition(cConfig *csconfig.Config) error {
 		}
 	}

+	if len(dataSources) == 0 {
+		return fmt.Errorf("no datasource enabled")
+	}
+
 	return nil
 }

--- a/docker/README.md
+++ b/docker/README.md
@ -138,10 +138,18 @@ agents on each machine that runs the protected applications, and a LAPI that
 gathers all signals from agents and communicates with the `central API`.

 ## Register a new agent with LAPI
+
+Without TLS authentication:
+
 ```shell
 docker exec -it crowdsec_lapi_container_name cscli machines add agent_user_name --password agent_password
 ```

+With TLS authentication:
+
+Agents are automatically registered and don't need a username or password. The
+agents' names are derived from the IP address from which they connect.
+
 ## Run an agent connected to LAPI

 Add the following environment variables to the docker run command:
@ -163,13 +171,20 @@ https://docs.crowdsec.net/docs/user_guides/bouncers_configuration/

 ### Automatic Bouncer Registration

-You can automatically register bouncers with the crowdsec container at startup, using environment variables or Docker secrets. You cannot use this process to update an existing bouncer without first deleting it.
+Without TLS authentication:
+
+You can register bouncers with the crowdsec container at startup, using environment variables or Docker secrets. You cannot use this process to update an existing bouncer without first deleting it.

 To use environment variables, they should be in the format `BOUNCER_KEY_<name>=<key>`. e.g. `BOUNCER_KEY_nginx=mysecretkey12345`.

 To use Docker secrets, the secret should be named `bouncer_key_<name>` with a content of `<key>`. e.g. `bouncer_key_nginx` with content `mysecretkey12345`.

-A bouncer key can be any string but we recommend an alphanumeric value for consistency with crowdsec-generated keys and avoid problems with escaping special characters.
+A bouncer key can be any string but we recommend an alphanumeric value for consistency with the crowdsec-generated keys and to avoid problems with escaping special characters.
+
+With TLS authentication:
+
+Bouncers are automatically registered and don't need an API key. The
+bouncers' names are derived from the IP address from which they connect.

 ## Console
 We provide a web-based interface to get more from Crowdsec: https://docs.crowdsec.net/docs/console
@ -183,22 +198,33 @@ Using binds rather than named volumes ([complete explanation here](https://docs.
 # Reference
 ## Environment Variables

+Note for persistent configurations (i.e. bind mount or volumes): when a
+variable is set, its value may be written to the appropriate file (usually
+config.yaml) each time the container is run.
+
+
 | Variable                | Default                   | Description |
 | ----------------------- | ------------------------- | ----------- |
 | `CONFIG_FILE`           | `/etc/crowdsec/config.yaml` | Configuration file location |
-| `DSN`                   | | Process a single source in time-machine: `-e DSN="file:///var/log/toto.log"` or `-e DSN="cloudwatch:///your/group/path:stream_name?profile=dev&backlog=16h"` or `-e DSN="journalctl://filters=_SYSTEMD_UNIT=ssh.service"` |
-| `TYPE`                  | | [`Labels.type`](https://docs.crowdsec.net/Crowdsec/v1/references/acquisition/) for file in time-machine: `-e TYPE="<type>"` |
-| `TEST_MODE`             | false | Don't run the service, only test the configuration: `-e TEST_MODE=true` |
-| `TZ`                    | | Set the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) to ensure the logs have a local timestamp. |
-| `LOCAL_API_URL`         | `http://0.0.0.0:8080` | The LAPI URL, you need to change this when `DISABLE_LOCAL_API` is true: `-e LOCAL_API_URL="http://lapi-address:8080"` |
 | `DISABLE_AGENT`         | false | Disable the agent, run a LAPI-only container |
 | `DISABLE_LOCAL_API`     | false | Disable LAPI, run an agent-only container |
 | `DISABLE_ONLINE_API`    | false | Disable online API registration for signal sharing |
-| `CUSTOM_HOSTNAME`       | localhost | Custom hostname for LAPI registration (with agent and LAPI on the same container) |
+| `TEST_MODE`             | false | Don't run the service, only test the configuration: `-e TEST_MODE=true` |
+| `TZ`                    | | Set the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) to ensure the logs have a local timestamp. |
+| `LOCAL_API_URL`         | `http://0.0.0.0:8080` | The LAPI URL, you need to change this when `DISABLE_LOCAL_API` is true: `-e LOCAL_API_URL="http://lapi-address:8080"` |
 | `PLUGIN_DIR`            | `/usr/local/lib/crowdsec/plugins/` | Directory for plugins: `-e PLUGIN_DIR="<path>"` |
-| `BOUNCER_KEY_<name>`    | | Register a bouncer with the name `<name>` and a key equal to the value of the environment variable. |
 | `METRICS_PORT`          | 6060 | Port to expose Prometheus metrics |
+|                         | | |
+| __LAPI__                | | (useless with DISABLE_LOCAL_API) |
 | `USE_WAL`               | false | Enable Write-Ahead Logging with SQLite |
+| `CUSTOM_HOSTNAME`       | localhost | Name for the local agent (running in the container with LAPI) |
+|                         | | |
+| __Agent__               | | (these don't work with DISABLE_AGENT) |
+| `TYPE`                  | | [`Labels.type`](https://docs.crowdsec.net/Crowdsec/v1/references/acquisition/) for file in time-machine: `-e TYPE="<type>"` |
+| `DSN`                   | | Process a single source in time-machine: `-e DSN="file:///var/log/toto.log"` or `-e DSN="cloudwatch:///your/group/path:stream_name?profile=dev&backlog=16h"` or `-e DSN="journalctl://filters=_SYSTEMD_UNIT=ssh.service"` |
+|                         | | |
+| __Bouncers__            | | |
+| `BOUNCER_KEY_<name>`    | | Register a bouncer with the name `<name>` and a key equal to the value of the environment variable. |
 |                         | | |
 | __Console__             | | |
 | `ENROLL_KEY`            | | Enroll key retrieved from [the console](https://app.crowdsec.net/) to enroll the instance. |
@ -209,18 +235,23 @@ Using binds rather than named volumes ([complete explanation here](https://docs.
 | `AGENT_USERNAME`        | | Agent username (to register if is LAPI or to use if it's an agent): `-e AGENT_USERNAME="machine_id"` |
 | `AGENT_PASSWORD`        | | Agent password (to register if is LAPI or to use if it's an agent): `-e AGENT_PASSWORD="machine_password"` |
 |                         | | |
-| __TLS Auth/encryption   | | |
-| `USE_TLS`               | false | Enable TLS on the LAPI |
-| `CERT_FILE`             | /etc/ssl/cert.pem | TLS Certificate path |
-| `KEY_FILE`              | /etc/ssl/key.pem | TLS Key path |
-| `CACERT_FILE`           | | CA certificate |
+| __TLS Encryption__      | | |
+| `USE_TLS`               | false | Enable TLS encryption (either as a LAPI or agent) |
+| `CACERT_FILE`           | | CA certificate bundle (for self-signed certificates) |
+| `INSECURE_SKIP_VERIFY`  | | Skip LAPI certificate validation |
+| `LAPI_CERT_FILE`        | | LAPI TLS Certificate path |
+| `LAPI_KEY_FILE`         | | LAPI TLS Key path |
+|                         | | |
+| __TLS Authentication__  | | (these require USE_TLS=true) |
+| `CLIENT_CERT_FILE`      | | Client TLS Certificate path (enable TLS authentication) |
+| `CLIENT_KEY_FILE`       | | Client TLS Key path |
 | `AGENTS_ALLOWED_OU`     | agent-ou | OU values allowed for agents, separated by comma |
 | `BOUNCERS_ALLOWED_OU`   | bouncer-ou | OU values allowed for bouncers, separated by comma |
 |                         | | |
 | __Hub management__      | | |
 | `COLLECTIONS`           | | Collections to install, separated by space: `-e COLLECTIONS="crowdsecurity/linux crowdsecurity/apache2"` |
-| `SCENARIOS`             | | Scenarios to install, separated by space |
 | `PARSERS`               | | Parsers to install, separated by space |
+| `SCENARIOS`             | | Scenarios to install, separated by space |
 | `POSTOVERFLOWS`         | | Postoverflows to install, separated by space |
 | `DISABLE_COLLECTIONS`   | | Collections to remove, separated by space: `-e DISABLE_COLLECTIONS="crowdsecurity/linux crowdsecurity/nginx"` |
 | `DISABLE_PARSERS`       | | Parsers to remove, separated by space |
@ -231,6 +262,10 @@ Using binds rather than named volumes ([complete explanation here](https://docs.
 | `LEVEL_INFO`            | false | Force INFO level for the container log |
 | `LEVEL_DEBUG`           | false | Force DEBUG level for the container log |
 | `LEVEL_TRACE`           | false | Force TRACE level (VERY verbose) for the container log |
+|                         | | |
+| __Developer options__   | | |
+| `CI_TESTING`            | false | Used during functional tests |
+| `DEBUG`                 | false | Trace the entrypoint |

 ## Volumes

--- a/docker/config.yaml
+++ b/docker/config.yaml
@ -25,14 +25,10 @@ db_config:
  log_level: info
  type: sqlite
  db_path: /var/lib/crowdsec/data/crowdsec.db
-  #user: 
-  #password:
-  #db_name:
-  #host:
-  #port:
  flush:
    max_items: 5000
    max_age: 7d
+  use_wal: false
 api:
  client:
    insecure_skip_verify: false
@ -45,10 +41,12 @@ api:
      - 127.0.0.1
      - ::1
    online_client: # Central API credentials (to push signals and receive bad IPs)
-      #credentials_path: /etc/crowdsec/online_api_credentials.yaml
-#    tls:
-#      cert_file: /etc/crowdsec/ssl/cert.pem
-#      key_file: /etc/crowdsec/ssl/key.pem
+    #credentials_path: /etc/crowdsec/online_api_credentials.yaml
+    tls:
+      agents_allowed_ou:
+        - agent-ou
+      bouncers_allowed_ou:
+        - bouncer-ou
 prometheus:
  enabled: true
  level: full
--- a/docker/docker_start.sh
+++ b/docker/docker_start.sh
@ -6,8 +6,7 @@
 set -e
 shopt -s inherit_errexit

-#- HELPER FUNCTIONS ----------------#
-
+# match true, TRUE, True, tRuE, etc.
 istrue() {
  case "$(echo "$1" | tr '[:upper:]' '[:lower:]')" in
    true) return 0 ;;
@ -23,6 +22,23 @@ isfalse() {
    fi
 }

+if istrue "$DEBUG"; then
+    set -x
+    export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
+fi
+
+if istrue "$CI_TESTING"; then
+    echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" >/etc/machine-id
+fi
+
+#- DEFAULTS -----------------------#
+
+export CONFIG_FILE="${CONFIG_FILE:=/etc/crowdsec/config.yaml}"
+export CUSTOM_HOSTNAME="${CUSTOM_HOSTNAME:=localhost}"
+
+#- HELPER FUNCTIONS ----------------#
+
+# csv2yaml <string>
 # generate a yaml list from a comma-separated string of values
 csv2yaml() {
    [ -z "$1" ] && return
@ -34,6 +50,8 @@ cscli() {
    command cscli -c "$CONFIG_FILE" "$@"
 }

+# conf_get <key> [file_path]
+# retrieve a value from a file (by default $CONFIG_FILE)
 conf_get() {
    if [ $# -ge 2 ]; then
        yq e "$1" "$2"
@ -42,16 +60,68 @@ conf_get() {
    fi
 }

+# conf_set <yq_expression> [file_path]
+# evaluate a yq command (by default on $CONFIG_FILE),
+# create the file if it doesn't exist
 conf_set() {
    if [ $# -ge 2 ]; then
-        yq e "$1" -i "$2"
+        YAML_FILE="$2"
    else
-        yq e "$1" -i "$CONFIG_FILE"
+        YAML_FILE="$CONFIG_FILE"
+    fi
+    YAML_CONTENT=$(cat "$YAML_FILE" 2>/dev/null || true)
+    NEW_CONTENT=$(echo "$YAML_CONTENT" | yq e "$1")
+    echo "$NEW_CONTENT" | install -m 0600 /dev/stdin "$YAML_FILE"
+}
+
+# conf_set_if(): used to update the configuration
+# only if a given variable is provided
+# conf_set_if "$VAR" <yq_expression> [file_path]
+conf_set_if() {
+    if [ "$1" != "" ]; then
+        shift
+        conf_set "$@"
    fi
 }

+# register_bouncer <bouncer_name> <bouncer_key>
+register_bouncer() {
+  if ! cscli bouncers list -o json | sed '/^ *"name"/!d;s/^ *"name": "\(.*\)",/\1/' | grep -q "^${1}$"; then
+      if cscli bouncers add "$1" -k "$2" > /dev/null; then
+          echo "Registered bouncer for $1"
+      else
+          echo "Failed to register bouncer for $1"
+      fi
+  fi
+}
+
+# Call cscli to manage objects ignoring taint errors
+# $1 can be collections, parsers, etc.
+# $2 can be install, remove, upgrade
+# $3 is a list of object names separated by space
+cscli_if_clean() {
+    # loop over all objects
+    for obj in $3; do
+        if cscli "$1" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then
+            echo "Object $1/$obj is tainted, skipping"
+        else
+            cscli "$1" "$2" "$obj"
+        fi
+    done
+}
+
 #-----------------------------------#

+if [ -n "$CERT_FILE" ] || [ -n "$KEY_FILE" ] ; then
+    printf '%b' '\033[0;33m'
+    echo "Warning: the variables CERT_FILE and KEY_FILE have been deprecated." >&2
+    echo "Please use LAPI_CERT_FILE and LAPI_KEY_FILE insted." >&2
+    echo "The old variables will be removed in a future release." >&2
+    printf '%b' '\033[0m'
+    LAPI_CERT_FILE=${LAPI_CERT_FILE:-$CERT_FILE}
+    LAPI_KEY_FILE=${LAPI_KEY_FILE:-$KEY_FILE}
+fi
+
 # Check and prestage databases
 for geodb in GeoLite2-ASN.mmdb GeoLite2-City.mmdb; do
    # We keep the pre-populated geoip databases in /staging instead of /var,
@ -84,52 +154,65 @@ elif [ -n "$USE_WAL" ] && isfalse "$USE_WAL"; then
    conf_set '.db_config.use_wal = false'
 fi

-# regenerate local agent credentials (ignore if agent is disabled)
-if isfalse "$DISABLE_AGENT"; then
-    if isfalse "$DISABLE_LOCAL_API"; then
-        echo "Regenerate local agent credentials"
-        cscli machines delete "$CUSTOM_HOSTNAME" 2>/dev/null || true
-        # shellcheck disable=SC2086
-        cscli machines add "$CUSTOM_HOSTNAME" --auto --url "$LOCAL_API_URL"
-    fi
-
-    lapi_credentials_path=$(conf_get '.api.client.credentials_path')
-
-    if istrue "$USE_TLS"; then
-        install -m 0600 /dev/null "$lapi_credentials_path"
-        conf_set '
-            .url = strenv(LOCAL_API_URL) |
-            .ca_cert_path = strenv(CACERT_FILE) |
-            .key_path = strenv(KEY_FILE) |
-            .cert_path = strenv(CERT_FILE)
-        ' "$lapi_credentials_path"
-    elif [ "$AGENT_USERNAME" != "" ]; then
-        install -m 0600 /dev/null "$lapi_credentials_path"
-        conf_set '
-            .url = strenv(LOCAL_API_URL) |
-            .login = strenv(AGENT_USERNAME) |
-            .password = strenv(AGENT_PASSWORD)
-        ' "$lapi_credentials_path"
-    fi
-fi
-
+# regenerate local agent credentials (even if agent is disabled, cscli needs a
+# connection to the API)
+cscli machines delete "$CUSTOM_HOSTNAME" 2>/dev/null || true
 if isfalse "$DISABLE_LOCAL_API"; then
-    echo "Check if lapi needs to automatically register an agent"
+    if isfalse "$USE_TLS" || [ "$CLIENT_CERT_FILE" = "" ]; then
+        echo "Regenerate local agent credentials"
+        cscli machines add "$CUSTOM_HOSTNAME" --auto
+    fi

-    # pre-registration is not needed with TLS
-    if isfalse "$USE_TLS" && [ "$AGENT_USERNAME" != "" ] && [ "$AGENT_PASSWORD" != "" ] ; then
-        # shellcheck disable=SC2086
-        cscli machines add "$AGENT_USERNAME" --password "$AGENT_PASSWORD" --url "$LOCAL_API_URL"
+    echo "Check if lapi needs to register an additional agent"
+    # pre-registration is not needed with TLS authentication, but we can have TLS transport with user/pw
+    if [ "$AGENT_USERNAME" != "" ] && [ "$AGENT_PASSWORD" != "" ] ; then
+        # re-register because pw may have been changed
+        cscli machines add "$AGENT_USERNAME" --password "$AGENT_PASSWORD" -f /dev/null --force
        echo "Agent registered to lapi"
    fi
 fi

+# ----------------
+
+lapi_credentials_path=$(conf_get '.api.client.credentials_path')
+
+conf_set_if "$LOCAL_API_URL" '.url = strenv(LOCAL_API_URL)' "$lapi_credentials_path"
+
+if istrue "$DISABLE_LOCAL_API"; then
+    # we only use the envvars that are actually defined
+    # in case of persistent configuration
+    conf_set_if "$AGENT_USERNAME" '.login = strenv(AGENT_USERNAME)' "$lapi_credentials_path"
+    conf_set_if "$AGENT_PASSWORD" '.password = strenv(AGENT_PASSWORD)' "$lapi_credentials_path"
+fi
+
+conf_set_if "$INSECURE_SKIP_VERIFY" '.api.client.insecure_skip_verify = env(INSECURE_SKIP_VERIFY)'
+
+# agent-only containers still require USE_TLS
+if istrue "$USE_TLS"; then
+    # shellcheck disable=SC2153
+    conf_set_if "$CACERT_FILE" '.ca_cert_path = strenv(CACERT_FILE)' "$lapi_credentials_path"
+    conf_set_if "$CLIENT_KEY_FILE" '.key_path = strenv(CLIENT_KEY_FILE)' "$lapi_credentials_path"
+    conf_set_if "$CLIENT_CERT_FILE" '.cert_path = strenv(CLIENT_CERT_FILE)' "$lapi_credentials_path"
+else
+    conf_set '
+        del(.ca_cert_path) |
+        del(.key_path) |
+        del(.cert_path)
+    ' "$lapi_credentials_path"
+fi
+
+if istrue "$DISABLE_ONLINE_API"; then
+    conf_set 'del(.api.server.online_client)'
+fi
+
 # registration to online API for signal push
-if isfalse "$DISABLE_ONLINE_API" && [ "$CONFIG_FILE" == "/etc/crowdsec/config.yaml" ] ; then
+if isfalse "$DISABLE_ONLINE_API" ; then
+    CONFIG_DIR=$(conf_get '.config_paths.config_dir')
    config_exists=$(conf_get '.api.server.online_client | has("credentials_path")')
    if isfalse "$config_exists"; then
-        conf_set '.api.server.online_client = {"credentials_path": "/etc/crowdsec/online_api_credentials.yaml"}'
-        cscli capi register > /etc/crowdsec/online_api_credentials.yaml
+        export CONFIG_DIR
+        conf_set '.api.server.online_client = {"credentials_path": strenv(CONFIG_DIR) + "/online_api_credentials.yaml"}'
+        cscli capi register > "$CONFIG_DIR/online_api_credentials.yaml"
        echo "Registration to online API done"
    fi
 fi
@ -158,84 +241,75 @@ if [ "$GID" != "" ]; then
    fi
 fi

+# XXX only with LAPI
 if istrue "$USE_TLS"; then
    agents_allowed_yaml=$(csv2yaml "$AGENTS_ALLOWED_OU") \
    bouncers_allowed_yaml=$(csv2yaml "$BOUNCERS_ALLOWED_OU") \
-    conf_set '
-        .api.server.tls.ca_cert_path = strenv(CACERT_FILE) |
-        .api.server.tls.cert_file = strenv(CERT_FILE) |
-        .api.server.tls.key_file = strenv(KEY_FILE) |
-        .api.server.tls.bouncers_allowed_ou = env(bouncers_allowed_yaml) |
-        .api.server.tls.agents_allowed_ou = env(agents_allowed_yaml) |
-        ... comments=""
-        '
+    conf_set_if "$CACERT_FILE" '.api.server.tls.ca_cert_path = strenv(CACERT_FILE)'
+    conf_set_if "$LAPI_CERT_FILE" '.api.server.tls.cert_file = strenv(LAPI_CERT_FILE)'
+    conf_set_if "$LAPI_KEY_FILE" '.api.server.tls.key_file = strenv(LAPI_KEY_FILE)'
+    conf_set_if "$BOUNCERS_ALLOWED_OU" '.api.server.tls.bouncers_allowed_ou = env(bouncers_allowed_yaml)'
+    conf_set_if "$AGENTS_ALLOWED_OU" '.api.server.tls.agents_allowed_ou = env(agents_allowed_yaml)'
+else
+    conf_set 'del(.api.server.tls)'
 fi

-conf_set ".config_paths.plugin_dir = strenv(PLUGIN_DIR)"
+conf_set_if "$PLUGIN_DIR" '.config_paths.plugin_dir = strenv(PLUGIN_DIR)'

 ## Install collections, parsers, scenarios & postoverflows
 cscli hub update
-cscli collections upgrade crowdsecurity/linux || true
-cscli parsers upgrade crowdsecurity/whitelists || true
-cscli parsers install crowdsecurity/docker-logs || true
+
+cscli_if_clean collections upgrade crowdsecurity/linux
+cscli_if_clean parsers upgrade crowdsecurity/whitelists
+cscli_if_clean parsers install crowdsecurity/docker-logs

 if [ "$COLLECTIONS" != "" ]; then
    # shellcheck disable=SC2086
-    cscli collections install $COLLECTIONS
+    cscli_if_clean collections install "$COLLECTIONS"
 fi

 if [ "$PARSERS" != "" ]; then
    # shellcheck disable=SC2086
-    cscli parsers install $PARSERS
+    cscli_if_clean parsers install "$PARSERS"
 fi

 if [ "$SCENARIOS" != "" ]; then
    # shellcheck disable=SC2086
-    cscli scenarios install $SCENARIOS
+    cscli_if_clean scenarios install "$SCENARIOS"
 fi

 if [ "$POSTOVERFLOWS" != "" ]; then
    # shellcheck disable=SC2086
-    cscli postoverflows install $POSTOVERFLOWS
+    cscli_if_clean postoverflows install "$POSTOVERFLOWS"
 fi

 ## Remove collections, parsers, scenarios & postoverflows
 if [ "$DISABLE_COLLECTIONS" != "" ]; then
    # shellcheck disable=SC2086
-    cscli collections remove $DISABLE_COLLECTIONS
+    cscli_if_clean collections remove "$DISABLE_COLLECTIONS"
 fi

 if [ "$DISABLE_PARSERS" != "" ]; then
    # shellcheck disable=SC2086
-    cscli parsers remove $DISABLE_PARSERS
+    cscli_if_clean parsers remove "$DISABLE_PARSERS"
 fi

 if [ "$DISABLE_SCENARIOS" != "" ]; then
    # shellcheck disable=SC2086
-    cscli scenarios remove $DISABLE_SCENARIOS
+    cscli_if_clean scenarios remove "$DISABLE_SCENARIOS"
 fi

 if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
    # shellcheck disable=SC2086
-    cscli postoverflows remove $DISABLE_POSTOVERFLOWS
+    cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS"
 fi

-register_bouncer() {
-  if ! cscli bouncers list -o json | sed '/^ *"name"/!d;s/^ *"name": "\(.*\)",/\1/' | grep -q "^${NAME}$"; then
-      if cscli bouncers add "${NAME}" -k "${KEY}" > /dev/null; then
-          echo "Registered bouncer for ${NAME}"
-      else
-          echo "Failed to register bouncer for ${NAME}"
-      fi
-  fi
-}
-
 ## Register bouncers via env
 for BOUNCER in $(compgen -A variable | grep -i BOUNCER_KEY); do
    KEY=$(printf '%s' "${!BOUNCER}")
-    NAME=$(printf '%s' "$BOUNCER" | cut -d_  -f2-)
+    NAME=$(printf '%s' "$BOUNCER" | cut -d_  -f3-)
    if [[ -n $KEY ]] && [[ -n $NAME ]]; then
-        register_bouncer
+        register_bouncer "$NAME" "$KEY"
    fi
 done

@ -245,7 +319,7 @@ for BOUNCER in /run/secrets/@(bouncer_key|BOUNCER_KEY)* ; do
    KEY=$(cat "${BOUNCER}")
    NAME=$(echo "${BOUNCER}" | awk -F "/" '{printf $NF}' | cut -d_  -f2-)
    if [[ -n $KEY ]] && [[ -n $NAME ]]; then    
-        register_bouncer
+        register_bouncer "$NAME" "$KEY"
    fi
 done
 shopt -u nullglob extglob
@ -287,7 +361,7 @@ if istrue "$LEVEL_INFO"; then
    ARGS="$ARGS -info"
 fi

-conf_set '.prometheus.listen_port=env(METRICS_PORT)'
+conf_set_if "$METRICS_PORT" '.prometheus.listen_port=env(METRICS_PORT)'

 # shellcheck disable=SC2086
 exec crowdsec $ARGS
--- a/pkg/acquisition/acquisition.go
+++ b/pkg/acquisition/acquisition.go
@ -25,6 +25,20 @@ import (
 	tomb "gopkg.in/tomb.v2"
 )

+type DataSourceUnavailableError struct {
+	Name string
+	Err error
+}
+
+func (e *DataSourceUnavailableError) Error() string {
+	return fmt.Sprintf("datasource '%s' is not available: %v", e.Name, e.Err)
+}
+
+func (e *DataSourceUnavailableError) Unwrap() error {
+	return e.Err
+}
+
+
 // The interface each datasource must implement
 type DataSource interface {
 	GetMetrics() []prometheus.Collector                         // Returns pointers to metrics that are managed by the module
@ -86,8 +100,11 @@ func GetDataSourceIface(dataSourceType string) DataSource {
 	return nil
 }

+// DataSourceConfigure creates and returns a DataSource object from a configuration,
+// if the configuration is not valid it returns an error.
+// If the datasource can't be run (eg. journalctl not available), it still returns an error which
+// can be checked for the appropriate action.
 func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataSource, error) {
-
 	//we dump it back to []byte, because we want to decode the yaml blob twice :
 	//once to DataSourceCommonCfg, and then later to the dedicated type of the datasource
 	yamlConfig, err := yaml.Marshal(commonConfig)
@ -112,7 +129,7 @@ func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataS
 		subLogger := clog.WithFields(customLog)
 		/* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */
 		if err := dataSrc.CanRun(); err != nil {
-			return nil, errors.Wrapf(err, "datasource %s cannot be run", commonConfig.Source)
+			return nil, &DataSourceUnavailableError{Name: commonConfig.Source, Err: err}
 		}
 		/* configure the actual datasource */
 		if err := dataSrc.Configure(yamlConfig, subLogger); err != nil {
@ -179,10 +196,11 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
 		}
 		dec := yaml.NewDecoder(yamlFile)
 		dec.SetStrict(true)
+		idx := -1
 		for {
 			var sub configuration.DataSourceCommonCfg
-			var idx int
 			err = dec.Decode(&sub)
+			idx += 1
 			if err != nil {
 				if ! errors.Is(err, io.EOF) {
 					return nil, errors.Wrapf(err, "failed to yaml decode %s", acquisFile)
@ -199,7 +217,6 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
 			if len(sub.Labels) == 0 {
 				if sub.Source == "" {
 					log.Debugf("skipping empty item in %s", acquisFile)
-					idx += 1
 					continue
 				}
 				return nil, fmt.Errorf("missing labels in %s (position: %d)", acquisFile, idx)
@ -212,10 +229,14 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
 			}
 			src, err := DataSourceConfigure(sub)
 			if err != nil {
+				var dserr *DataSourceUnavailableError
+				if errors.As(err, &dserr) {
+					log.Error(err)
+					continue
+				}
 				return nil, errors.Wrapf(err, "while configuring datasource of type %s from %s (position: %d)", sub.Source, acquisFile, idx)
 			}
 			sources = append(sources, *src)
-			idx += 1
 		}
 	}
 	return sources, nil
--- a/pkg/acquisition/acquisition_test.go
+++ b/pkg/acquisition/acquisition_test.go
@ -171,7 +171,7 @@ log_level: debug
 source: mock_cant_run
 wowo: ajsajasjas
 `,
-			ExpectedError: "datasource mock_cant_run cannot be run: can't run bro",
+			ExpectedError: "datasource 'mock_cant_run' is not available: can't run bro",
 		},
 	}

--- a/pkg/acquisition/modules/syslog/internal/parser/rfc3164/parse_test.go
+++ b/pkg/acquisition/modules/syslog/internal/parser/rfc3164/parse_test.go
@ -54,7 +54,7 @@ func TestTimestamp(t *testing.T) {
 		currentYear bool
 	}{
 		{"May 20 09:33:54", "0000-05-20T09:33:54Z", "", false},
-		{"May 20 09:33:54", "2022-05-20T09:33:54Z", "", true},
+		{"May 20 09:33:54", "2023-05-20T09:33:54Z", "", true},
 		{"May 20 09:33:54 2022", "2022-05-20T09:33:54Z", "", false},
 		{"May  1 09:33:54 2022", "2022-05-01T09:33:54Z", "", false},
 		{"May 01 09:33:54 2021", "2021-05-01T09:33:54Z", "", true},
@ -257,7 +257,7 @@ func TestParse(t *testing.T) {
 		},
 		{
 			"<12>May 20 09:33:54 UDMPRO,a2edd0c6ae48,udm-1.10.0.3686 kernel: foo", expected{
-				Timestamp: time.Date(2022, time.May, 20, 9, 33, 54, 0, time.UTC),
+				Timestamp: time.Date(2023, time.May, 20, 9, 33, 54, 0, time.UTC),
 				Hostname:  "UDMPRO,a2edd0c6ae48,udm-1.10.0.3686",
 				Tag:       "kernel",
 				PID:       "",
--- a/pkg/apiclient/client.go
+++ b/pkg/apiclient/client.go
@ -53,8 +53,8 @@ func NewClient(config *Config) (*ApiClient, error) {
 		UpdateScenario: config.UpdateScenario,
 	}
 	tlsconfig := tls.Config{InsecureSkipVerify: InsecureSkipVerify}
+	tlsconfig.RootCAs = CaCertPool
 	if Cert != nil {
-		tlsconfig.RootCAs = CaCertPool
 		tlsconfig.Certificates = []tls.Certificate{*Cert}
 	}
 	http.DefaultTransport.(*http.Transport).TLSClientConfig = &tlsconfig
@ -75,8 +75,8 @@ func NewDefaultClient(URL *url.URL, prefix string, userAgent string, client *htt
 		client = &http.Client{}
 		if ht, ok := http.DefaultTransport.(*http.Transport); ok {
 			tlsconfig := tls.Config{InsecureSkipVerify: InsecureSkipVerify}
+			tlsconfig.RootCAs = CaCertPool
 			if Cert != nil {
-				tlsconfig.RootCAs = CaCertPool
 				tlsconfig.Certificates = []tls.Certificate{*Cert}
 			}
 			ht.TLSClientConfig = &tlsconfig
--- a/pkg/csconfig/api.go
+++ b/pkg/csconfig/api.go
@ -81,7 +81,7 @@ func (l *LocalApiClientCfg) Load() error {
 		}
 	}

-	if l.Credentials.Login != "" && (l.Credentials.CACertPath != "" || l.Credentials.CertPath != "" || l.Credentials.KeyPath != "") {
+	if l.Credentials.Login != "" && (l.Credentials.CertPath != "" || l.Credentials.KeyPath != "") {
 		return fmt.Errorf("user/password authentication and TLS authentication are mutually exclusive")
 	}

@ -91,12 +91,7 @@ func (l *LocalApiClientCfg) Load() error {
 		apiclient.InsecureSkipVerify = *l.InsecureSkipVerify
 	}

-	if l.Credentials.CACertPath != "" && l.Credentials.CertPath != "" && l.Credentials.KeyPath != "" {
-		cert, err := tls.LoadX509KeyPair(l.Credentials.CertPath, l.Credentials.KeyPath)
-		if err != nil {
-			return errors.Wrapf(err, "failed to load api client certificate")
-		}
-
+	if l.Credentials.CACertPath != ""  {
 		caCert, err := os.ReadFile(l.Credentials.CACertPath)
 		if err != nil {
 			return errors.Wrapf(err, "failed to load cacert")
@ -104,10 +99,18 @@ func (l *LocalApiClientCfg) Load() error {

 		caCertPool := x509.NewCertPool()
 		caCertPool.AppendCertsFromPEM(caCert)
-
-		apiclient.Cert = &cert
 		apiclient.CaCertPool = caCertPool
 	}
+
+	if l.Credentials.CertPath != "" && l.Credentials.KeyPath != "" {
+		cert, err := tls.LoadX509KeyPair(l.Credentials.CertPath, l.Credentials.KeyPath)
+		if err != nil {
+			return errors.Wrapf(err, "failed to load api client certificate")
+		}
+
+		apiclient.Cert = &cert
+	}
+
 	return nil
 }

--- a/tests/bats/01_crowdsec.bats
+++ b/tests/bats/01_crowdsec.bats
@ -148,9 +148,10 @@ teardown() {
    rm -f "$ACQUIS_DIR"

    config_set '.common.log_media="stdout"'
-    run -124 --separate-stderr timeout 2s "${CROWDSEC}"
+    run -1 --separate-stderr timeout 2s "${CROWDSEC}"
    # check warning
-    assert_stderr_line --partial "no acquisition file found"
+    assert_stderr --partial "no acquisition file found"
+    assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
 }

@test "crowdsec (error if acquisition_path and acquisition_dir are not defined)" {
@ -163,19 +164,55 @@ teardown() {
    config_set '.crowdsec_service.acquisition_dir=""'

    config_set '.common.log_media="stdout"'
-    run -124 --separate-stderr timeout 2s "${CROWDSEC}"
+    run -1 --separate-stderr timeout 2s "${CROWDSEC}"
    # check warning
-    assert_stderr_line --partial "no acquisition_path or acquisition_dir specified"
+    assert_stderr --partial "no acquisition_path or acquisition_dir specified"
+    assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
 }

@test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" {
    ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
-    rm -f "$ACQUIS_YAML"
    config_set '.crowdsec_service.acquisition_path=""'

    ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir')
    mkdir -p "$ACQUIS_DIR"
-    touch "$ACQUIS_DIR"/foo.yaml
+    mv "$ACQUIS_YAML" "$ACQUIS_DIR"/foo.yaml

    run -124 --separate-stderr timeout 2s "${CROWDSEC}"
+
+    # now, if foo.yaml is empty instead, there won't be valid datasources.
+
+    cat /dev/null >"$ACQUIS_DIR"/foo.yaml
+
+    run --separate-stderr -1 timeout 2s "${CROWDSEC}"
+    assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
 }
+
+@test "crowdsec (disabled datasources)" {
+    config_set '.common.log_media="stdout"'
+
+    # a datasource cannot run - missing journalctl command
+
+    ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir')
+    mkdir -p "$ACQUIS_DIR"
+    cat >"$ACQUIS_DIR"/foo.yaml <<-EOT
+	source: journalctl
+	journalctl_filter:
+	- "_SYSTEMD_UNIT=ssh.service"
+	labels:
+	  type: syslog
+	EOT
+
+    run --separate-stderr -124 timeout 2s env PATH='' "${CROWDSEC}"
+    #shellcheck disable=SC2016
+    assert_stderr --partial 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in $PATH'
+
+    # if all datasources are disabled, crowdsec should exit
+
+    ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
+    rm -f "$ACQUIS_YAML"
+    config_set '.crowdsec_service.acquisition_path=""'
+
+    run --separate-stderr -1 timeout 2s env PATH='' "${CROWDSEC}"
+    assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
+ }
--- a/tests/bin/assert-crowdsec-not-running
+++ b/tests/bin/assert-crowdsec-not-running
@ -1,7 +1,7 @@
 #!/usr/bin/env bash

 is_crowdsec_running() {
-    PIDS=$(pgrep -x 'crowdsec|crowdsec.test|crowdsec.cover')
+    PIDS=$(pgrep -x 'crowdsec|crowdsec.test|crowdsec.cover' 2>/dev/null)
 }

 # The process can be slow, especially on CI and during test coverage.
Author	SHA1	Message	Date
mmetc	a0d69783cd	non-fatal error if some datasource can't be run (i.e. journalctl but systemd is missing) (#2310 ) This on the other hand, gives a new fatal error when there are no valid datasources. In the previous version, crowdsec kept running with just a warning if no acquisition yaml or dir were specified.	2023-06-27 10:46:25 +02:00
Manuel Sabban	5f71037b40	strip v prefix in tag name in azure-pipeline (#2048 ) (#2049 ) Co-authored-by: blotus <sebastien@crowdsec.net>	2023-02-09 14:27:11 +01:00
Manuel Sabban	c08b37426e	Fix azure pipeline (#2046 ) (#2047 ) * use the tag name from the predefined variable for azure pipeline (#2041) * fix typo when assigning build_version in azure pipeline (#2044) * Fix azure pipeline (#2046) --------- Co-authored-by: blotus <sebastien@crowdsec.net>	2023-02-09 13:15:36 +01:00
Manuel Sabban	358a4f49ea	Fix build windows (#2045 ) * use the tag name from the predefined variable for azure pipeline (#2041) * fix typo when assigning build_version in azure pipeline (#2044) --------- Co-authored-by: blotus <sebastien@crowdsec.net>	2023-02-09 11:01:11 +01:00
Manuel Sabban	97ee59ef71	use the tag name from the predefined variable for azure pipeline (#2041 ) (#2043 ) Co-authored-by: blotus <sebastien@crowdsec.net>	2023-02-09 10:23:07 +01:00
Yip Rui Fung	a9a2186a76	Fix docker_start.sh not properly handling env vars (#1993 ) For example, the COLLECTIONS environment variable is supposed to do a space separated list. But with the unquoted call to cscli_if_clean without quotes on the $COLLECTIONS environment variable, only the first entry is passed to it. As a result, only the first entry is installed. Would likely affect all call sites to cscli_if_clean	2023-01-15 09:04:50 +01:00
mmetc	8f400f95de	docker: add {VERSION}-slim tag to releases (#1977 )	2023-01-15 09:04:50 +01:00
mmetc	8d0af73a4f	docker: install gcc before yq (fix i386 build) (#1965 )	2023-01-03 16:49:25 +01:00
mmetc	ff6ade73ce	docker entrypoint/configuration fixes + refactoring (#1959 )	2023-01-03 14:45:33 +01:00
Marco Mariani	3ab6429199	fix parser test 2k23	2023-01-03 09:04:32 +01:00
mmetc	1b28792ae2	fix tls communication with lapi and user/pw auth (backport) (#1955 ) allow self-signed TLS encryption with user/pw auth docker: - remove defaults for certificate file locations - new envvar INSECURE_SKIP_VERIFY - register agent before TLS settings (cscli machine add removes them from the credentials file)	2022-12-29 21:59:30 +01:00
mmetc	6ef4217643	ci: remove hub dispatch, always full history when building, build version from git (#1942 )	2022-12-27 16:43:39 +01:00
mmetc	80e6a2e60b	ci: build msi package on prerelease action, explicit job dependency (#1941 )	2022-12-22 14:36:41 +01:00
mmetc	79aecac2da	ci: define job output (#1940 )	2022-12-22 11:06:05 +01:00
mmetc	b561a370cd	ci: authenticate when looking up release information (#1936 ) (#1939 )	2022-12-22 10:09:18 +01:00
mmetc	dbc06d430f	docker: separate CLIENT_* and LAPI_* variables for tls certificates (#1931 )	2022-12-19 09:50:42 +01:00
mmetc	86666f4b29	docker: fix/improve support for persistent configurations (#1915 ) (#1922 ) set all defaults in config.yaml and leave environment variables empty. This way when they are set we know that we must override the values in config.yaml. ignore tainted objects when calling install/upgrade/remove use_wal is false by default	2022-12-12 11:00:04 +01:00
mmetc	c996a218c1	docker/README: automatic registration with tls	2022-12-12 10:59:39 +01:00
mmetc	3b7a26e419	docker: correctly extract BOUNCER_KEY_* (fix #1912 ) (#1913 ) (#1920 )	2022-12-12 10:58:52 +01:00
mmetc	3366a31e93	set cscli log timestamp to 24h (#1917 ) (#1921 )	2022-12-12 10:01:11 +01:00