Compare commits
20 commits
master
...
releases/1
Author | SHA1 | Date | |
---|---|---|---|
|
a0d69783cd | ||
|
5f71037b40 | ||
|
c08b37426e | ||
|
358a4f49ea | ||
|
97ee59ef71 | ||
|
a9a2186a76 | ||
|
8f400f95de | ||
|
8d0af73a4f | ||
|
ff6ade73ce | ||
|
3ab6429199 | ||
|
1b28792ae2 | ||
|
6ef4217643 | ||
|
80e6a2e60b | ||
|
79aecac2da | ||
|
b561a370cd | ||
|
dbc06d430f | ||
|
86666f4b29 | ||
|
c996a218c1 | ||
|
3b7a26e419 | ||
|
3366a31e93 |
|
@ -1,3 +1,4 @@
|
||||||
# We include .git in the build context because excluding it would break the
|
# We include .git in the build context because excluding it would break the
|
||||||
# "make release" target, which uses git to retrieve the build version and tag.
|
# "make release" target, which uses git to retrieve the build version and tag.
|
||||||
#.git
|
#.git
|
||||||
|
/tests/ansible
|
||||||
|
|
21
.github/workflows/ci-windows-build-msi.yml
vendored
21
.github/workflows/ci-windows-build-msi.yml
vendored
|
@ -1,17 +1,11 @@
|
||||||
name: build-msi (windows)
|
name: build-msi (windows)
|
||||||
|
|
||||||
on:
|
on:
|
||||||
pull_request:
|
release:
|
||||||
branches:
|
types:
|
||||||
- master
|
- prereleased
|
||||||
- releases/**
|
|
||||||
paths-ignore:
|
|
||||||
- 'docs/**'
|
|
||||||
- 'mkdocs.yml'
|
|
||||||
- 'README.md'
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
|
|
||||||
build:
|
build:
|
||||||
name: Build
|
name: Build
|
||||||
runs-on: windows-2019
|
runs-on: windows-2019
|
||||||
|
@ -22,14 +16,9 @@ jobs:
|
||||||
go-version: 1.19
|
go-version: 1.19
|
||||||
id: go
|
id: go
|
||||||
- name: Check out code into the Go module directory
|
- name: Check out code into the Go module directory
|
||||||
uses: actions/checkout@v2
|
uses: actions/checkout@v3
|
||||||
- id: get_latest_release
|
|
||||||
uses: pozetroninc/github-action-get-latest-release@master
|
|
||||||
with:
|
with:
|
||||||
repository: crowdsecurity/crowdsec
|
fetch-depth: 0
|
||||||
excludes: draft
|
|
||||||
- id: set_release_in_env
|
|
||||||
run: echo "BUILD_VERSION=${{ steps.get_latest_release.outputs.release }}" >> $env:GITHUB_ENV
|
|
||||||
- name: Build
|
- name: Build
|
||||||
run: make windows_installer
|
run: make windows_installer
|
||||||
- name: Upload MSI
|
- name: Upload MSI
|
||||||
|
|
22
.github/workflows/dispatch_ci_hub.yaml
vendored
22
.github/workflows/dispatch_ci_hub.yaml
vendored
|
@ -1,22 +0,0 @@
|
||||||
name: Dispatch to hub when creating pre-release
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
# branches to consider in the event; optional, defaults to all
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
- releases/**
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
dispatch:
|
|
||||||
name: dispatch to hub-tests
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Repository Dispatch
|
|
||||||
uses: peter-evans/repository-dispatch@v1
|
|
||||||
if: ${{ github.repository_owner == 'crowdsecurity' }}
|
|
||||||
with:
|
|
||||||
token: ${{ secrets.DISPATCH_TOKEN }}
|
|
||||||
event-type: trigger_ci_hub
|
|
||||||
repository: crowdsecurity/hub
|
|
||||||
client-payload: '{"version": "${{ steps.keydb.outputs.release }}"}'
|
|
|
@ -1,24 +0,0 @@
|
||||||
name: Dispatch to hub when creating pre-release
|
|
||||||
|
|
||||||
on:
|
|
||||||
release:
|
|
||||||
types: prereleased
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
dispatch:
|
|
||||||
name: dispatch to hub-tests
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- id: keydb
|
|
||||||
uses: pozetroninc/github-action-get-latest-release@master
|
|
||||||
with:
|
|
||||||
owner: crowdsecurity
|
|
||||||
repo: crowdsec
|
|
||||||
excludes: prerelease, draft
|
|
||||||
- name: Repository Dispatch
|
|
||||||
uses: peter-evans/repository-dispatch@v1
|
|
||||||
with:
|
|
||||||
token: ${{ secrets.DISPATCH_TOKEN }}
|
|
||||||
event-type: create_branch
|
|
||||||
repository: crowdsecurity/hub
|
|
||||||
client-payload: '{"version": "${{ steps.keydb.outputs.release }}"}'
|
|
|
@ -1,24 +0,0 @@
|
||||||
name: Dispatch to hub when deleting pre-release
|
|
||||||
|
|
||||||
on:
|
|
||||||
release:
|
|
||||||
types: deleted
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
dispatch:
|
|
||||||
name: dispatch to hub-tests
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- id: keydb
|
|
||||||
uses: pozetroninc/github-action-get-latest-release@master
|
|
||||||
with:
|
|
||||||
owner: crowdsecurity
|
|
||||||
repo: crowdsec
|
|
||||||
excludes: prerelease, draft
|
|
||||||
- name: Repository Dispatch
|
|
||||||
uses: peter-evans/repository-dispatch@v1
|
|
||||||
with:
|
|
||||||
token: ${{ secrets.DISPATCH_TOKEN }}
|
|
||||||
event-type: delete_branch
|
|
||||||
repository: crowdsecurity/hub
|
|
||||||
client-payload: '{"version": "${{ steps.keydb.outputs.release }}"}'
|
|
|
@ -3,7 +3,8 @@ name: build
|
||||||
|
|
||||||
on:
|
on:
|
||||||
release:
|
release:
|
||||||
types: prereleased
|
types:
|
||||||
|
- prereleased
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build:
|
build:
|
||||||
|
@ -17,6 +18,8 @@ jobs:
|
||||||
id: go
|
id: go
|
||||||
- name: Check out code into the Go module directory
|
- name: Check out code into the Go module directory
|
||||||
uses: actions/checkout@v3
|
uses: actions/checkout@v3
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
- name: Build the binaries
|
- name: Build the binaries
|
||||||
run: make release
|
run: make release
|
||||||
- name: Upload to release
|
- name: Upload to release
|
||||||
|
@ -36,6 +39,8 @@ jobs:
|
||||||
id: go
|
id: go
|
||||||
- name: Check out code into the Go module directory
|
- name: Check out code into the Go module directory
|
||||||
uses: actions/checkout@v3
|
uses: actions/checkout@v3
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
- name: Build the binaries
|
- name: Build the binaries
|
||||||
run: |
|
run: |
|
||||||
make release BUILD_STATIC=yes
|
make release BUILD_STATIC=yes
|
||||||
|
|
|
@ -32,7 +32,7 @@ jobs:
|
||||||
TAGS_SLIM="${DOCKER_IMAGE}:${VERSION}-slim"
|
TAGS_SLIM="${DOCKER_IMAGE}:${VERSION}-slim"
|
||||||
if [[ ${{ github.event.action }} == released ]]; then
|
if [[ ${{ github.event.action }} == released ]]; then
|
||||||
TAGS=$TAGS,${DOCKER_IMAGE}:latest,${GHCR_IMAGE}:latest
|
TAGS=$TAGS,${DOCKER_IMAGE}:latest,${GHCR_IMAGE}:latest
|
||||||
TAGS_SLIM=$TAGS,${DOCKER_IMAGE}:slim
|
TAGS_SLIM=$TAGS_SLIM,${DOCKER_IMAGE}:slim
|
||||||
fi
|
fi
|
||||||
echo ::set-output name=version::${VERSION}
|
echo ::set-output name=version::${VERSION}
|
||||||
echo ::set-output name=tags::${TAGS}
|
echo ::set-output name=tags::${TAGS}
|
||||||
|
|
67
Dockerfile
67
Dockerfile
|
@ -1,3 +1,4 @@
|
||||||
|
# vim: set ft=dockerfile:
|
||||||
ARG BUILD_ENV=full
|
ARG BUILD_ENV=full
|
||||||
ARG GOVERSION=1.19
|
ARG GOVERSION=1.19
|
||||||
|
|
||||||
|
@ -9,82 +10,30 @@ COPY . .
|
||||||
|
|
||||||
# wizard.sh requires GNU coreutils
|
# wizard.sh requires GNU coreutils
|
||||||
RUN apk add --no-cache git gcc libc-dev make bash gettext binutils-gold coreutils && \
|
RUN apk add --no-cache git gcc libc-dev make bash gettext binutils-gold coreutils && \
|
||||||
|
echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
|
||||||
SYSTEM="docker" make clean release && \
|
SYSTEM="docker" make clean release && \
|
||||||
cd crowdsec-v* && \
|
cd crowdsec-v* && \
|
||||||
./wizard.sh --docker-mode && \
|
./wizard.sh --docker-mode && \
|
||||||
cd - && \
|
cd - >/dev/null && \
|
||||||
cscli hub update && \
|
cscli hub update && \
|
||||||
cscli collections install crowdsecurity/linux && \
|
cscli collections install crowdsecurity/linux && \
|
||||||
cscli parsers install crowdsecurity/whitelists
|
cscli parsers install crowdsecurity/whitelists && \
|
||||||
|
go install github.com/mikefarah/yq/v4@v4.30.6
|
||||||
|
|
||||||
FROM alpine:latest as build-slim
|
FROM alpine:latest as build-slim
|
||||||
|
|
||||||
RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata yq bash && \
|
RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata bash && \
|
||||||
mkdir -p /staging/etc/crowdsec && \
|
mkdir -p /staging/etc/crowdsec && \
|
||||||
mkdir -p /staging/var/lib/crowdsec && \
|
mkdir -p /staging/var/lib/crowdsec && \
|
||||||
mkdir -p /var/lib/crowdsec/data
|
mkdir -p /var/lib/crowdsec/data
|
||||||
|
|
||||||
|
COPY --from=build /go/bin/yq /usr/local/bin/yq
|
||||||
COPY --from=build /etc/crowdsec /staging/etc/crowdsec
|
COPY --from=build /etc/crowdsec /staging/etc/crowdsec
|
||||||
COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
|
COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
|
||||||
COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
|
COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
|
||||||
COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
|
COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
|
||||||
COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
|
COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
|
||||||
|
RUN yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml
|
||||||
ENV CONFIG_FILE=/etc/crowdsec/config.yaml
|
|
||||||
ENV LOCAL_API_URL=http://0.0.0.0:8080/
|
|
||||||
ENV CUSTOM_HOSTNAME=localhost
|
|
||||||
ENV PLUGIN_DIR=/usr/local/lib/crowdsec/plugins/
|
|
||||||
ENV DISABLE_AGENT=false
|
|
||||||
ENV DISABLE_LOCAL_API=false
|
|
||||||
ENV DISABLE_ONLINE_API=false
|
|
||||||
ENV DSN=
|
|
||||||
ENV TYPE=
|
|
||||||
ENV TEST_MODE=false
|
|
||||||
ENV USE_WAL=false
|
|
||||||
|
|
||||||
# register to app.crowdsec.net
|
|
||||||
|
|
||||||
ENV ENROLL_INSTANCE_NAME=
|
|
||||||
ENV ENROLL_KEY=
|
|
||||||
ENV ENROLL_TAGS=
|
|
||||||
|
|
||||||
# log verbosity
|
|
||||||
|
|
||||||
ENV LEVEL_TRACE=false
|
|
||||||
ENV LEVEL_DEBUG=false
|
|
||||||
ENV LEVEL_INFO=true
|
|
||||||
|
|
||||||
# TLS setup ----------------------------------- #
|
|
||||||
|
|
||||||
ENV AGENT_USERNAME=
|
|
||||||
ENV AGENT_PASSWORD=
|
|
||||||
|
|
||||||
# TLS setup ----------------------------------- #
|
|
||||||
|
|
||||||
ENV USE_TLS=false
|
|
||||||
ENV CA_CERT_PATH=
|
|
||||||
ENV CERT_FILE=/etc/ssl/cert.pem
|
|
||||||
ENV KEY_FILE=/etc/ssl/key.pem
|
|
||||||
# comma-separated list of allowed OU values for TLS bouncer certificates
|
|
||||||
ENV BOUNCERS_ALLOWED_OU=bouncer-ou
|
|
||||||
# comma-separated list of allowed OU values for TLS agent certificates
|
|
||||||
ENV AGENTS_ALLOWED_OU=agent-ou
|
|
||||||
|
|
||||||
# Install the following hub items --------------#
|
|
||||||
|
|
||||||
ENV COLLECTIONS=
|
|
||||||
ENV PARSERS=
|
|
||||||
ENV SCENARIOS=
|
|
||||||
ENV POSTOVERFLOWS=
|
|
||||||
|
|
||||||
# Uninstall the following hub items ------------#
|
|
||||||
|
|
||||||
ENV DISABLE_COLLECTIONS=
|
|
||||||
ENV DISABLE_PARSERS=
|
|
||||||
ENV DISABLE_SCENARIOS=
|
|
||||||
ENV DISABLE_POSTOVERFLOWS=
|
|
||||||
|
|
||||||
ENV METRICS_PORT=6060
|
|
||||||
|
|
||||||
ENTRYPOINT /bin/bash docker_start.sh
|
ENTRYPOINT /bin/bash docker_start.sh
|
||||||
|
|
||||||
|
|
|
@ -8,20 +8,27 @@ WORKDIR /go/src/crowdsec
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
|
ENV DEBIAN_FRONTEND=noninteractive
|
||||||
|
ENV DEBCONF_NOWARNINGS="yes"
|
||||||
|
|
||||||
# wizard.sh requires GNU coreutils
|
# wizard.sh requires GNU coreutils
|
||||||
RUN apt-get update && \
|
RUN apt-get update && \
|
||||||
apt-get install -y git gcc libc-dev make bash gettext binutils-gold coreutils tzdata && \
|
apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata && \
|
||||||
SYSTEM="docker" make release && \
|
echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
|
||||||
|
SYSTEM="docker" make clean release && \
|
||||||
cd crowdsec-v* && \
|
cd crowdsec-v* && \
|
||||||
./wizard.sh --docker-mode && \
|
./wizard.sh --docker-mode && \
|
||||||
cd - && \
|
cd - >/dev/null && \
|
||||||
cscli hub update && \
|
cscli hub update && \
|
||||||
cscli collections install crowdsecurity/linux && \
|
cscli collections install crowdsecurity/linux && \
|
||||||
cscli parsers install crowdsecurity/whitelists && \
|
cscli parsers install crowdsecurity/whitelists && \
|
||||||
go install github.com/mikefarah/yq/v4@latest
|
go install github.com/mikefarah/yq/v4@v4.30.6
|
||||||
|
|
||||||
FROM debian:bullseye-slim as build-slim
|
FROM debian:bullseye-slim as build-slim
|
||||||
|
|
||||||
|
ENV DEBIAN_FRONTEND=noninteractive
|
||||||
|
ENV DEBCONF_NOWARNINGS="yes"
|
||||||
|
|
||||||
RUN apt-get update && \
|
RUN apt-get update && \
|
||||||
apt-get install -y -q --install-recommends --no-install-suggests \
|
apt-get install -y -q --install-recommends --no-install-suggests \
|
||||||
procps \
|
procps \
|
||||||
|
@ -40,63 +47,9 @@ COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
|
||||||
COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
|
COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
|
||||||
COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
|
COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
|
||||||
COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
|
COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
|
||||||
RUN yq eval -i ".plugin_config.group = \"nogroup\"" /staging/etc/crowdsec/config.yaml
|
RUN yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml && \
|
||||||
|
yq eval -i ".plugin_config.group = \"nogroup\"" /staging/etc/crowdsec/config.yaml
|
||||||
|
|
||||||
ENV CONFIG_FILE=/etc/crowdsec/config.yaml
|
|
||||||
ENV LOCAL_API_URL=http://0.0.0.0:8080/
|
|
||||||
ENV CUSTOM_HOSTNAME=localhost
|
|
||||||
ENV PLUGIN_DIR=/usr/local/lib/crowdsec/plugins/
|
|
||||||
ENV DISABLE_AGENT=false
|
|
||||||
ENV DISABLE_LOCAL_API=false
|
|
||||||
ENV DISABLE_ONLINE_API=false
|
|
||||||
ENV DSN=
|
|
||||||
ENV TYPE=
|
|
||||||
ENV TEST_MODE=false
|
|
||||||
ENV USE_WAL=false
|
|
||||||
|
|
||||||
# register to app.crowdsec.net
|
|
||||||
|
|
||||||
ENV ENROLL_INSTANCE_NAME=
|
|
||||||
ENV ENROLL_KEY=
|
|
||||||
ENV ENROLL_TAGS=
|
|
||||||
|
|
||||||
# log verbosity
|
|
||||||
|
|
||||||
ENV LEVEL_TRACE=false
|
|
||||||
ENV LEVEL_DEBUG=false
|
|
||||||
ENV LEVEL_INFO=true
|
|
||||||
|
|
||||||
# TLS setup ----------------------------------- #
|
|
||||||
|
|
||||||
ENV AGENT_USERNAME=
|
|
||||||
ENV AGENT_PASSWORD=
|
|
||||||
|
|
||||||
# TLS setup ----------------------------------- #
|
|
||||||
|
|
||||||
ENV USE_TLS=false
|
|
||||||
ENV CA_CERT_PATH=
|
|
||||||
ENV CERT_FILE=/etc/ssl/cert.pem
|
|
||||||
ENV KEY_FILE=/etc/ssl/key.pem
|
|
||||||
# comma-separated list of allowed OU values for TLS bouncer certificates
|
|
||||||
ENV BOUNCERS_ALLOWED_OU=bouncer-ou
|
|
||||||
# comma-separated list of allowed OU values for TLS agent certificates
|
|
||||||
ENV AGENTS_ALLOWED_OU=agent-ou
|
|
||||||
|
|
||||||
# Install the following hub items --------------#
|
|
||||||
|
|
||||||
ENV COLLECTIONS=
|
|
||||||
ENV PARSERS=
|
|
||||||
ENV SCENARIOS=
|
|
||||||
ENV POSTOVERFLOWS=
|
|
||||||
|
|
||||||
# Uninstall the following hub items ------------#
|
|
||||||
|
|
||||||
ENV DISABLE_COLLECTIONS=
|
|
||||||
ENV DISABLE_PARSERS=
|
|
||||||
ENV DISABLE_SCENARIOS=
|
|
||||||
ENV DISABLE_POSTOVERFLOWS=
|
|
||||||
|
|
||||||
ENV METRICS_PORT=6060
|
|
||||||
|
|
||||||
ENTRYPOINT /bin/bash docker_start.sh
|
ENTRYPOINT /bin/bash docker_start.sh
|
||||||
|
|
||||||
|
|
|
@ -56,25 +56,26 @@ stages:
|
||||||
--input "**/*.exe" --config (Join-Path -Path $(Agent.TempDirectory) -ChildPath "appsettings.json") `
|
--input "**/*.exe" --config (Join-Path -Path $(Agent.TempDirectory) -ChildPath "appsettings.json") `
|
||||||
--user $(CodeSigningUser) --secret '$(CodeSigningPassword)'
|
--user $(CodeSigningUser) --secret '$(CodeSigningPassword)'
|
||||||
displayName: "Sign Crowdsec binaries + plugins"
|
displayName: "Sign Crowdsec binaries + plugins"
|
||||||
|
|
||||||
- pwsh: |
|
- pwsh: |
|
||||||
$build_version=(git describe --tags (git rev-list --tags --max-count=1)).Substring(1)
|
$build_version=$env:BUILD_SOURCEBRANCHNAME
|
||||||
|
if ($build_version.StartsWith("v"))
|
||||||
|
{
|
||||||
|
$build_version = $build_version.Substring(1)
|
||||||
|
}
|
||||||
if ($build_version.Contains("-"))
|
if ($build_version.Contains("-"))
|
||||||
{
|
{
|
||||||
$build_version = $build_version.Substring(0, $build_version.IndexOf("-"))
|
$build_version = $build_version.Substring(0, $build_version.IndexOf("-"))
|
||||||
}
|
}
|
||||||
.\make_installer.ps1 -version $build_version
|
|
||||||
Write-Host "##vso[task.setvariable variable=BuildVersion;isOutput=true]$build_version"
|
Write-Host "##vso[task.setvariable variable=BuildVersion;isOutput=true]$build_version"
|
||||||
|
displayName: GetCrowdsecVersion
|
||||||
|
name: GetCrowdsecVersion
|
||||||
|
- pwsh: |
|
||||||
|
.\make_installer.ps1 -version '$(GetCrowdsecVersion.BuildVersion)'
|
||||||
displayName: "Build Crowdsec MSI"
|
displayName: "Build Crowdsec MSI"
|
||||||
name: BuildMSI
|
name: BuildMSI
|
||||||
|
|
||||||
- pwsh: |
|
- pwsh: |
|
||||||
$build_version=(git describe --tags (git rev-list --tags --max-count=1)).Substring(1)
|
.\make_chocolatey.ps1 -version '$(GetCrowdsecVersion.BuildVersion)'
|
||||||
if ($build_version.Contains("-"))
|
|
||||||
{
|
|
||||||
$build_version = $build_version.Substring(0, $build_version.IndexOf("-"))
|
|
||||||
}
|
|
||||||
.\make_chocolatey.ps1 -version $build_version
|
|
||||||
displayName: "Build Chocolatey nupkg"
|
displayName: "Build Chocolatey nupkg"
|
||||||
|
|
||||||
- pwsh: |
|
- pwsh: |
|
||||||
|
@ -85,14 +86,14 @@ stages:
|
||||||
|
|
||||||
- task: PublishBuildArtifacts@1
|
- task: PublishBuildArtifacts@1
|
||||||
inputs:
|
inputs:
|
||||||
PathtoPublish: '$(Build.Repository.LocalPath)\\crowdsec_$(BuildMSI.BuildVersion).msi'
|
PathtoPublish: '$(Build.Repository.LocalPath)\\crowdsec_$(GetCrowdsecVersion.BuildVersion).msi'
|
||||||
ArtifactName: 'crowdsec.msi'
|
ArtifactName: 'crowdsec.msi'
|
||||||
publishLocation: 'Container'
|
publishLocation: 'Container'
|
||||||
displayName: "Upload MSI artifact"
|
displayName: "Upload MSI artifact"
|
||||||
|
|
||||||
- task: PublishBuildArtifacts@1
|
- task: PublishBuildArtifacts@1
|
||||||
inputs:
|
inputs:
|
||||||
PathtoPublish: '$(Build.Repository.LocalPath)\\windows\\Chocolatey\\crowdsec\\crowdsec.$(BuildMSI.BuildVersion).nupkg'
|
PathtoPublish: '$(Build.Repository.LocalPath)\\windows\\Chocolatey\\crowdsec\\crowdsec.$(GetCrowdsecVersion.BuildVersion).nupkg'
|
||||||
ArtifactName: 'crowdsec.nupkg'
|
ArtifactName: 'crowdsec.nupkg'
|
||||||
publishLocation: 'Container'
|
publishLocation: 'Container'
|
||||||
displayName: "Upload nupkg artifact"
|
displayName: "Upload nupkg artifact"
|
||||||
|
|
|
@ -52,7 +52,7 @@ func initConfig() {
|
||||||
} else if err_lvl {
|
} else if err_lvl {
|
||||||
log.SetLevel(log.ErrorLevel)
|
log.SetLevel(log.ErrorLevel)
|
||||||
}
|
}
|
||||||
logFormatter := &log.TextFormatter{TimestampFormat: "02-01-2006 03:04:05 PM", FullTimestamp: true}
|
logFormatter := &log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}
|
||||||
log.SetFormatter(logFormatter)
|
log.SetFormatter(logFormatter)
|
||||||
|
|
||||||
if !inSlice(os.Args[1], NoNeedConfig) {
|
if !inSlice(os.Args[1], NoNeedConfig) {
|
||||||
|
|
|
@ -23,22 +23,22 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
// Populate cwhub package tools
|
// Populate cwhub package tools
|
||||||
if err := cwhub.GetHubIdx(cConfig.Hub); err != nil {
|
if err = cwhub.GetHubIdx(cConfig.Hub); err != nil {
|
||||||
return &parser.Parsers{}, fmt.Errorf("Failed to load hub index : %s", err)
|
return &parser.Parsers{}, fmt.Errorf("while loading hub index : %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start loading configs
|
// Start loading configs
|
||||||
csParsers := newParsers()
|
csParsers := newParsers()
|
||||||
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
|
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
|
||||||
return &parser.Parsers{}, fmt.Errorf("Failed to load parsers: %s", err)
|
return nil, fmt.Errorf("while loading parsers: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := LoadBuckets(cConfig); err != nil {
|
if err := LoadBuckets(cConfig); err != nil {
|
||||||
return &parser.Parsers{}, fmt.Errorf("Failed to load scenarios: %s", err)
|
return nil, fmt.Errorf("while loading scenarios: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := LoadAcquisition(cConfig); err != nil {
|
if err := LoadAcquisition(cConfig); err != nil {
|
||||||
return &parser.Parsers{}, fmt.Errorf("Error while loading acquisition config : %s", err)
|
return nil, fmt.Errorf("while loading acquisition config: %s", err)
|
||||||
}
|
}
|
||||||
return csParsers, nil
|
return csParsers, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -157,6 +157,10 @@ func LoadAcquisition(cConfig *csconfig.Config) error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(dataSources) == 0 {
|
||||||
|
return fmt.Errorf("no datasource enabled")
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -138,10 +138,18 @@ agents on each machine that runs the protected applications, and a LAPI that
|
||||||
gathers all signals from agents and communicates with the `central API`.
|
gathers all signals from agents and communicates with the `central API`.
|
||||||
|
|
||||||
## Register a new agent with LAPI
|
## Register a new agent with LAPI
|
||||||
|
|
||||||
|
Without TLS authentication:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
docker exec -it crowdsec_lapi_container_name cscli machines add agent_user_name --password agent_password
|
docker exec -it crowdsec_lapi_container_name cscli machines add agent_user_name --password agent_password
|
||||||
```
|
```
|
||||||
|
|
||||||
|
With TLS authentication:
|
||||||
|
|
||||||
|
Agents are automatically registered and don't need a username or password. The
|
||||||
|
agents' names are derived from the IP address from which they connect.
|
||||||
|
|
||||||
## Run an agent connected to LAPI
|
## Run an agent connected to LAPI
|
||||||
|
|
||||||
Add the following environment variables to the docker run command:
|
Add the following environment variables to the docker run command:
|
||||||
|
@ -163,13 +171,20 @@ https://docs.crowdsec.net/docs/user_guides/bouncers_configuration/
|
||||||
|
|
||||||
### Automatic Bouncer Registration
|
### Automatic Bouncer Registration
|
||||||
|
|
||||||
You can automatically register bouncers with the crowdsec container at startup, using environment variables or Docker secrets. You cannot use this process to update an existing bouncer without first deleting it.
|
Without TLS authentication:
|
||||||
|
|
||||||
|
You can register bouncers with the crowdsec container at startup, using environment variables or Docker secrets. You cannot use this process to update an existing bouncer without first deleting it.
|
||||||
|
|
||||||
To use environment variables, they should be in the format `BOUNCER_KEY_<name>=<key>`. e.g. `BOUNCER_KEY_nginx=mysecretkey12345`.
|
To use environment variables, they should be in the format `BOUNCER_KEY_<name>=<key>`. e.g. `BOUNCER_KEY_nginx=mysecretkey12345`.
|
||||||
|
|
||||||
To use Docker secrets, the secret should be named `bouncer_key_<name>` with a content of `<key>`. e.g. `bouncer_key_nginx` with content `mysecretkey12345`.
|
To use Docker secrets, the secret should be named `bouncer_key_<name>` with a content of `<key>`. e.g. `bouncer_key_nginx` with content `mysecretkey12345`.
|
||||||
|
|
||||||
A bouncer key can be any string but we recommend an alphanumeric value for consistency with crowdsec-generated keys and avoid problems with escaping special characters.
|
A bouncer key can be any string but we recommend an alphanumeric value for consistency with the crowdsec-generated keys and to avoid problems with escaping special characters.
|
||||||
|
|
||||||
|
With TLS authentication:
|
||||||
|
|
||||||
|
Bouncers are automatically registered and don't need an API key. The
|
||||||
|
bouncers' names are derived from the IP address from which they connect.
|
||||||
|
|
||||||
## Console
|
## Console
|
||||||
We provide a web-based interface to get more from Crowdsec: https://docs.crowdsec.net/docs/console
|
We provide a web-based interface to get more from Crowdsec: https://docs.crowdsec.net/docs/console
|
||||||
|
@ -183,22 +198,33 @@ Using binds rather than named volumes ([complete explanation here](https://docs.
|
||||||
# Reference
|
# Reference
|
||||||
## Environment Variables
|
## Environment Variables
|
||||||
|
|
||||||
|
Note for persistent configurations (i.e. bind mount or volumes): when a
|
||||||
|
variable is set, its value may be written to the appropriate file (usually
|
||||||
|
config.yaml) each time the container is run.
|
||||||
|
|
||||||
|
|
||||||
| Variable | Default | Description |
|
| Variable | Default | Description |
|
||||||
| ----------------------- | ------------------------- | ----------- |
|
| ----------------------- | ------------------------- | ----------- |
|
||||||
| `CONFIG_FILE` | `/etc/crowdsec/config.yaml` | Configuration file location |
|
| `CONFIG_FILE` | `/etc/crowdsec/config.yaml` | Configuration file location |
|
||||||
| `DSN` | | Process a single source in time-machine: `-e DSN="file:///var/log/toto.log"` or `-e DSN="cloudwatch:///your/group/path:stream_name?profile=dev&backlog=16h"` or `-e DSN="journalctl://filters=_SYSTEMD_UNIT=ssh.service"` |
|
|
||||||
| `TYPE` | | [`Labels.type`](https://docs.crowdsec.net/Crowdsec/v1/references/acquisition/) for file in time-machine: `-e TYPE="<type>"` |
|
|
||||||
| `TEST_MODE` | false | Don't run the service, only test the configuration: `-e TEST_MODE=true` |
|
|
||||||
| `TZ` | | Set the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) to ensure the logs have a local timestamp. |
|
|
||||||
| `LOCAL_API_URL` | `http://0.0.0.0:8080` | The LAPI URL, you need to change this when `DISABLE_LOCAL_API` is true: `-e LOCAL_API_URL="http://lapi-address:8080"` |
|
|
||||||
| `DISABLE_AGENT` | false | Disable the agent, run a LAPI-only container |
|
| `DISABLE_AGENT` | false | Disable the agent, run a LAPI-only container |
|
||||||
| `DISABLE_LOCAL_API` | false | Disable LAPI, run an agent-only container |
|
| `DISABLE_LOCAL_API` | false | Disable LAPI, run an agent-only container |
|
||||||
| `DISABLE_ONLINE_API` | false | Disable online API registration for signal sharing |
|
| `DISABLE_ONLINE_API` | false | Disable online API registration for signal sharing |
|
||||||
| `CUSTOM_HOSTNAME` | localhost | Custom hostname for LAPI registration (with agent and LAPI on the same container) |
|
| `TEST_MODE` | false | Don't run the service, only test the configuration: `-e TEST_MODE=true` |
|
||||||
|
| `TZ` | | Set the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) to ensure the logs have a local timestamp. |
|
||||||
|
| `LOCAL_API_URL` | `http://0.0.0.0:8080` | The LAPI URL, you need to change this when `DISABLE_LOCAL_API` is true: `-e LOCAL_API_URL="http://lapi-address:8080"` |
|
||||||
| `PLUGIN_DIR` | `/usr/local/lib/crowdsec/plugins/` | Directory for plugins: `-e PLUGIN_DIR="<path>"` |
|
| `PLUGIN_DIR` | `/usr/local/lib/crowdsec/plugins/` | Directory for plugins: `-e PLUGIN_DIR="<path>"` |
|
||||||
| `BOUNCER_KEY_<name>` | | Register a bouncer with the name `<name>` and a key equal to the value of the environment variable. |
|
|
||||||
| `METRICS_PORT` | 6060 | Port to expose Prometheus metrics |
|
| `METRICS_PORT` | 6060 | Port to expose Prometheus metrics |
|
||||||
|
| | | |
|
||||||
|
| __LAPI__ | | (useless with DISABLE_LOCAL_API) |
|
||||||
| `USE_WAL` | false | Enable Write-Ahead Logging with SQLite |
|
| `USE_WAL` | false | Enable Write-Ahead Logging with SQLite |
|
||||||
|
| `CUSTOM_HOSTNAME` | localhost | Name for the local agent (running in the container with LAPI) |
|
||||||
|
| | | |
|
||||||
|
| __Agent__ | | (these don't work with DISABLE_AGENT) |
|
||||||
|
| `TYPE` | | [`Labels.type`](https://docs.crowdsec.net/Crowdsec/v1/references/acquisition/) for file in time-machine: `-e TYPE="<type>"` |
|
||||||
|
| `DSN` | | Process a single source in time-machine: `-e DSN="file:///var/log/toto.log"` or `-e DSN="cloudwatch:///your/group/path:stream_name?profile=dev&backlog=16h"` or `-e DSN="journalctl://filters=_SYSTEMD_UNIT=ssh.service"` |
|
||||||
|
| | | |
|
||||||
|
| __Bouncers__ | | |
|
||||||
|
| `BOUNCER_KEY_<name>` | | Register a bouncer with the name `<name>` and a key equal to the value of the environment variable. |
|
||||||
| | | |
|
| | | |
|
||||||
| __Console__ | | |
|
| __Console__ | | |
|
||||||
| `ENROLL_KEY` | | Enroll key retrieved from [the console](https://app.crowdsec.net/) to enroll the instance. |
|
| `ENROLL_KEY` | | Enroll key retrieved from [the console](https://app.crowdsec.net/) to enroll the instance. |
|
||||||
|
@ -209,18 +235,23 @@ Using binds rather than named volumes ([complete explanation here](https://docs.
|
||||||
| `AGENT_USERNAME` | | Agent username (to register if is LAPI or to use if it's an agent): `-e AGENT_USERNAME="machine_id"` |
|
| `AGENT_USERNAME` | | Agent username (to register if is LAPI or to use if it's an agent): `-e AGENT_USERNAME="machine_id"` |
|
||||||
| `AGENT_PASSWORD` | | Agent password (to register if is LAPI or to use if it's an agent): `-e AGENT_PASSWORD="machine_password"` |
|
| `AGENT_PASSWORD` | | Agent password (to register if is LAPI or to use if it's an agent): `-e AGENT_PASSWORD="machine_password"` |
|
||||||
| | | |
|
| | | |
|
||||||
| __TLS Auth/encryption | | |
|
| __TLS Encryption__ | | |
|
||||||
| `USE_TLS` | false | Enable TLS on the LAPI |
|
| `USE_TLS` | false | Enable TLS encryption (either as a LAPI or agent) |
|
||||||
| `CERT_FILE` | /etc/ssl/cert.pem | TLS Certificate path |
|
| `CACERT_FILE` | | CA certificate bundle (for self-signed certificates) |
|
||||||
| `KEY_FILE` | /etc/ssl/key.pem | TLS Key path |
|
| `INSECURE_SKIP_VERIFY` | | Skip LAPI certificate validation |
|
||||||
| `CACERT_FILE` | | CA certificate |
|
| `LAPI_CERT_FILE` | | LAPI TLS Certificate path |
|
||||||
|
| `LAPI_KEY_FILE` | | LAPI TLS Key path |
|
||||||
|
| | | |
|
||||||
|
| __TLS Authentication__ | | (these require USE_TLS=true) |
|
||||||
|
| `CLIENT_CERT_FILE` | | Client TLS Certificate path (enable TLS authentication) |
|
||||||
|
| `CLIENT_KEY_FILE` | | Client TLS Key path |
|
||||||
| `AGENTS_ALLOWED_OU` | agent-ou | OU values allowed for agents, separated by comma |
|
| `AGENTS_ALLOWED_OU` | agent-ou | OU values allowed for agents, separated by comma |
|
||||||
| `BOUNCERS_ALLOWED_OU` | bouncer-ou | OU values allowed for bouncers, separated by comma |
|
| `BOUNCERS_ALLOWED_OU` | bouncer-ou | OU values allowed for bouncers, separated by comma |
|
||||||
| | | |
|
| | | |
|
||||||
| __Hub management__ | | |
|
| __Hub management__ | | |
|
||||||
| `COLLECTIONS` | | Collections to install, separated by space: `-e COLLECTIONS="crowdsecurity/linux crowdsecurity/apache2"` |
|
| `COLLECTIONS` | | Collections to install, separated by space: `-e COLLECTIONS="crowdsecurity/linux crowdsecurity/apache2"` |
|
||||||
| `SCENARIOS` | | Scenarios to install, separated by space |
|
|
||||||
| `PARSERS` | | Parsers to install, separated by space |
|
| `PARSERS` | | Parsers to install, separated by space |
|
||||||
|
| `SCENARIOS` | | Scenarios to install, separated by space |
|
||||||
| `POSTOVERFLOWS` | | Postoverflows to install, separated by space |
|
| `POSTOVERFLOWS` | | Postoverflows to install, separated by space |
|
||||||
| `DISABLE_COLLECTIONS` | | Collections to remove, separated by space: `-e DISABLE_COLLECTIONS="crowdsecurity/linux crowdsecurity/nginx"` |
|
| `DISABLE_COLLECTIONS` | | Collections to remove, separated by space: `-e DISABLE_COLLECTIONS="crowdsecurity/linux crowdsecurity/nginx"` |
|
||||||
| `DISABLE_PARSERS` | | Parsers to remove, separated by space |
|
| `DISABLE_PARSERS` | | Parsers to remove, separated by space |
|
||||||
|
@ -231,6 +262,10 @@ Using binds rather than named volumes ([complete explanation here](https://docs.
|
||||||
| `LEVEL_INFO` | false | Force INFO level for the container log |
|
| `LEVEL_INFO` | false | Force INFO level for the container log |
|
||||||
| `LEVEL_DEBUG` | false | Force DEBUG level for the container log |
|
| `LEVEL_DEBUG` | false | Force DEBUG level for the container log |
|
||||||
| `LEVEL_TRACE` | false | Force TRACE level (VERY verbose) for the container log |
|
| `LEVEL_TRACE` | false | Force TRACE level (VERY verbose) for the container log |
|
||||||
|
| | | |
|
||||||
|
| __Developer options__ | | |
|
||||||
|
| `CI_TESTING` | false | Used during functional tests |
|
||||||
|
| `DEBUG` | false | Trace the entrypoint |
|
||||||
|
|
||||||
## Volumes
|
## Volumes
|
||||||
|
|
||||||
|
|
|
@ -25,14 +25,10 @@ db_config:
|
||||||
log_level: info
|
log_level: info
|
||||||
type: sqlite
|
type: sqlite
|
||||||
db_path: /var/lib/crowdsec/data/crowdsec.db
|
db_path: /var/lib/crowdsec/data/crowdsec.db
|
||||||
#user:
|
|
||||||
#password:
|
|
||||||
#db_name:
|
|
||||||
#host:
|
|
||||||
#port:
|
|
||||||
flush:
|
flush:
|
||||||
max_items: 5000
|
max_items: 5000
|
||||||
max_age: 7d
|
max_age: 7d
|
||||||
|
use_wal: false
|
||||||
api:
|
api:
|
||||||
client:
|
client:
|
||||||
insecure_skip_verify: false
|
insecure_skip_verify: false
|
||||||
|
@ -46,9 +42,11 @@ api:
|
||||||
- ::1
|
- ::1
|
||||||
online_client: # Central API credentials (to push signals and receive bad IPs)
|
online_client: # Central API credentials (to push signals and receive bad IPs)
|
||||||
#credentials_path: /etc/crowdsec/online_api_credentials.yaml
|
#credentials_path: /etc/crowdsec/online_api_credentials.yaml
|
||||||
# tls:
|
tls:
|
||||||
# cert_file: /etc/crowdsec/ssl/cert.pem
|
agents_allowed_ou:
|
||||||
# key_file: /etc/crowdsec/ssl/key.pem
|
- agent-ou
|
||||||
|
bouncers_allowed_ou:
|
||||||
|
- bouncer-ou
|
||||||
prometheus:
|
prometheus:
|
||||||
enabled: true
|
enabled: true
|
||||||
level: full
|
level: full
|
||||||
|
|
|
@ -6,8 +6,7 @@
|
||||||
set -e
|
set -e
|
||||||
shopt -s inherit_errexit
|
shopt -s inherit_errexit
|
||||||
|
|
||||||
#- HELPER FUNCTIONS ----------------#
|
# match true, TRUE, True, tRuE, etc.
|
||||||
|
|
||||||
istrue() {
|
istrue() {
|
||||||
case "$(echo "$1" | tr '[:upper:]' '[:lower:]')" in
|
case "$(echo "$1" | tr '[:upper:]' '[:lower:]')" in
|
||||||
true) return 0 ;;
|
true) return 0 ;;
|
||||||
|
@ -23,6 +22,23 @@ isfalse() {
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if istrue "$DEBUG"; then
|
||||||
|
set -x
|
||||||
|
export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
|
||||||
|
fi
|
||||||
|
|
||||||
|
if istrue "$CI_TESTING"; then
|
||||||
|
echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" >/etc/machine-id
|
||||||
|
fi
|
||||||
|
|
||||||
|
#- DEFAULTS -----------------------#
|
||||||
|
|
||||||
|
export CONFIG_FILE="${CONFIG_FILE:=/etc/crowdsec/config.yaml}"
|
||||||
|
export CUSTOM_HOSTNAME="${CUSTOM_HOSTNAME:=localhost}"
|
||||||
|
|
||||||
|
#- HELPER FUNCTIONS ----------------#
|
||||||
|
|
||||||
|
# csv2yaml <string>
|
||||||
# generate a yaml list from a comma-separated string of values
|
# generate a yaml list from a comma-separated string of values
|
||||||
csv2yaml() {
|
csv2yaml() {
|
||||||
[ -z "$1" ] && return
|
[ -z "$1" ] && return
|
||||||
|
@ -34,6 +50,8 @@ cscli() {
|
||||||
command cscli -c "$CONFIG_FILE" "$@"
|
command cscli -c "$CONFIG_FILE" "$@"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# conf_get <key> [file_path]
|
||||||
|
# retrieve a value from a file (by default $CONFIG_FILE)
|
||||||
conf_get() {
|
conf_get() {
|
||||||
if [ $# -ge 2 ]; then
|
if [ $# -ge 2 ]; then
|
||||||
yq e "$1" "$2"
|
yq e "$1" "$2"
|
||||||
|
@ -42,16 +60,68 @@ conf_get() {
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# conf_set <yq_expression> [file_path]
|
||||||
|
# evaluate a yq command (by default on $CONFIG_FILE),
|
||||||
|
# create the file if it doesn't exist
|
||||||
conf_set() {
|
conf_set() {
|
||||||
if [ $# -ge 2 ]; then
|
if [ $# -ge 2 ]; then
|
||||||
yq e "$1" -i "$2"
|
YAML_FILE="$2"
|
||||||
else
|
else
|
||||||
yq e "$1" -i "$CONFIG_FILE"
|
YAML_FILE="$CONFIG_FILE"
|
||||||
|
fi
|
||||||
|
YAML_CONTENT=$(cat "$YAML_FILE" 2>/dev/null || true)
|
||||||
|
NEW_CONTENT=$(echo "$YAML_CONTENT" | yq e "$1")
|
||||||
|
echo "$NEW_CONTENT" | install -m 0600 /dev/stdin "$YAML_FILE"
|
||||||
|
}
|
||||||
|
|
||||||
|
# conf_set_if(): used to update the configuration
|
||||||
|
# only if a given variable is provided
|
||||||
|
# conf_set_if "$VAR" <yq_expression> [file_path]
|
||||||
|
conf_set_if() {
|
||||||
|
if [ "$1" != "" ]; then
|
||||||
|
shift
|
||||||
|
conf_set "$@"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# register_bouncer <bouncer_name> <bouncer_key>
|
||||||
|
register_bouncer() {
|
||||||
|
if ! cscli bouncers list -o json | sed '/^ *"name"/!d;s/^ *"name": "\(.*\)",/\1/' | grep -q "^${1}$"; then
|
||||||
|
if cscli bouncers add "$1" -k "$2" > /dev/null; then
|
||||||
|
echo "Registered bouncer for $1"
|
||||||
|
else
|
||||||
|
echo "Failed to register bouncer for $1"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Call cscli to manage objects ignoring taint errors
|
||||||
|
# $1 can be collections, parsers, etc.
|
||||||
|
# $2 can be install, remove, upgrade
|
||||||
|
# $3 is a list of object names separated by space
|
||||||
|
cscli_if_clean() {
|
||||||
|
# loop over all objects
|
||||||
|
for obj in $3; do
|
||||||
|
if cscli "$1" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then
|
||||||
|
echo "Object $1/$obj is tainted, skipping"
|
||||||
|
else
|
||||||
|
cscli "$1" "$2" "$obj"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
#-----------------------------------#
|
#-----------------------------------#
|
||||||
|
|
||||||
|
if [ -n "$CERT_FILE" ] || [ -n "$KEY_FILE" ] ; then
|
||||||
|
printf '%b' '\033[0;33m'
|
||||||
|
echo "Warning: the variables CERT_FILE and KEY_FILE have been deprecated." >&2
|
||||||
|
echo "Please use LAPI_CERT_FILE and LAPI_KEY_FILE insted." >&2
|
||||||
|
echo "The old variables will be removed in a future release." >&2
|
||||||
|
printf '%b' '\033[0m'
|
||||||
|
LAPI_CERT_FILE=${LAPI_CERT_FILE:-$CERT_FILE}
|
||||||
|
LAPI_KEY_FILE=${LAPI_KEY_FILE:-$KEY_FILE}
|
||||||
|
fi
|
||||||
|
|
||||||
# Check and prestage databases
|
# Check and prestage databases
|
||||||
for geodb in GeoLite2-ASN.mmdb GeoLite2-City.mmdb; do
|
for geodb in GeoLite2-ASN.mmdb GeoLite2-City.mmdb; do
|
||||||
# We keep the pre-populated geoip databases in /staging instead of /var,
|
# We keep the pre-populated geoip databases in /staging instead of /var,
|
||||||
|
@ -84,52 +154,65 @@ elif [ -n "$USE_WAL" ] && isfalse "$USE_WAL"; then
|
||||||
conf_set '.db_config.use_wal = false'
|
conf_set '.db_config.use_wal = false'
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# regenerate local agent credentials (ignore if agent is disabled)
|
# regenerate local agent credentials (even if agent is disabled, cscli needs a
|
||||||
if isfalse "$DISABLE_AGENT"; then
|
# connection to the API)
|
||||||
if isfalse "$DISABLE_LOCAL_API"; then
|
|
||||||
echo "Regenerate local agent credentials"
|
|
||||||
cscli machines delete "$CUSTOM_HOSTNAME" 2>/dev/null || true
|
cscli machines delete "$CUSTOM_HOSTNAME" 2>/dev/null || true
|
||||||
# shellcheck disable=SC2086
|
|
||||||
cscli machines add "$CUSTOM_HOSTNAME" --auto --url "$LOCAL_API_URL"
|
|
||||||
fi
|
|
||||||
|
|
||||||
lapi_credentials_path=$(conf_get '.api.client.credentials_path')
|
|
||||||
|
|
||||||
if istrue "$USE_TLS"; then
|
|
||||||
install -m 0600 /dev/null "$lapi_credentials_path"
|
|
||||||
conf_set '
|
|
||||||
.url = strenv(LOCAL_API_URL) |
|
|
||||||
.ca_cert_path = strenv(CACERT_FILE) |
|
|
||||||
.key_path = strenv(KEY_FILE) |
|
|
||||||
.cert_path = strenv(CERT_FILE)
|
|
||||||
' "$lapi_credentials_path"
|
|
||||||
elif [ "$AGENT_USERNAME" != "" ]; then
|
|
||||||
install -m 0600 /dev/null "$lapi_credentials_path"
|
|
||||||
conf_set '
|
|
||||||
.url = strenv(LOCAL_API_URL) |
|
|
||||||
.login = strenv(AGENT_USERNAME) |
|
|
||||||
.password = strenv(AGENT_PASSWORD)
|
|
||||||
' "$lapi_credentials_path"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if isfalse "$DISABLE_LOCAL_API"; then
|
if isfalse "$DISABLE_LOCAL_API"; then
|
||||||
echo "Check if lapi needs to automatically register an agent"
|
if isfalse "$USE_TLS" || [ "$CLIENT_CERT_FILE" = "" ]; then
|
||||||
|
echo "Regenerate local agent credentials"
|
||||||
|
cscli machines add "$CUSTOM_HOSTNAME" --auto
|
||||||
|
fi
|
||||||
|
|
||||||
# pre-registration is not needed with TLS
|
echo "Check if lapi needs to register an additional agent"
|
||||||
if isfalse "$USE_TLS" && [ "$AGENT_USERNAME" != "" ] && [ "$AGENT_PASSWORD" != "" ] ; then
|
# pre-registration is not needed with TLS authentication, but we can have TLS transport with user/pw
|
||||||
# shellcheck disable=SC2086
|
if [ "$AGENT_USERNAME" != "" ] && [ "$AGENT_PASSWORD" != "" ] ; then
|
||||||
cscli machines add "$AGENT_USERNAME" --password "$AGENT_PASSWORD" --url "$LOCAL_API_URL"
|
# re-register because pw may have been changed
|
||||||
|
cscli machines add "$AGENT_USERNAME" --password "$AGENT_PASSWORD" -f /dev/null --force
|
||||||
echo "Agent registered to lapi"
|
echo "Agent registered to lapi"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# ----------------
|
||||||
|
|
||||||
|
lapi_credentials_path=$(conf_get '.api.client.credentials_path')
|
||||||
|
|
||||||
|
conf_set_if "$LOCAL_API_URL" '.url = strenv(LOCAL_API_URL)' "$lapi_credentials_path"
|
||||||
|
|
||||||
|
if istrue "$DISABLE_LOCAL_API"; then
|
||||||
|
# we only use the envvars that are actually defined
|
||||||
|
# in case of persistent configuration
|
||||||
|
conf_set_if "$AGENT_USERNAME" '.login = strenv(AGENT_USERNAME)' "$lapi_credentials_path"
|
||||||
|
conf_set_if "$AGENT_PASSWORD" '.password = strenv(AGENT_PASSWORD)' "$lapi_credentials_path"
|
||||||
|
fi
|
||||||
|
|
||||||
|
conf_set_if "$INSECURE_SKIP_VERIFY" '.api.client.insecure_skip_verify = env(INSECURE_SKIP_VERIFY)'
|
||||||
|
|
||||||
|
# agent-only containers still require USE_TLS
|
||||||
|
if istrue "$USE_TLS"; then
|
||||||
|
# shellcheck disable=SC2153
|
||||||
|
conf_set_if "$CACERT_FILE" '.ca_cert_path = strenv(CACERT_FILE)' "$lapi_credentials_path"
|
||||||
|
conf_set_if "$CLIENT_KEY_FILE" '.key_path = strenv(CLIENT_KEY_FILE)' "$lapi_credentials_path"
|
||||||
|
conf_set_if "$CLIENT_CERT_FILE" '.cert_path = strenv(CLIENT_CERT_FILE)' "$lapi_credentials_path"
|
||||||
|
else
|
||||||
|
conf_set '
|
||||||
|
del(.ca_cert_path) |
|
||||||
|
del(.key_path) |
|
||||||
|
del(.cert_path)
|
||||||
|
' "$lapi_credentials_path"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if istrue "$DISABLE_ONLINE_API"; then
|
||||||
|
conf_set 'del(.api.server.online_client)'
|
||||||
|
fi
|
||||||
|
|
||||||
# registration to online API for signal push
|
# registration to online API for signal push
|
||||||
if isfalse "$DISABLE_ONLINE_API" && [ "$CONFIG_FILE" == "/etc/crowdsec/config.yaml" ] ; then
|
if isfalse "$DISABLE_ONLINE_API" ; then
|
||||||
|
CONFIG_DIR=$(conf_get '.config_paths.config_dir')
|
||||||
config_exists=$(conf_get '.api.server.online_client | has("credentials_path")')
|
config_exists=$(conf_get '.api.server.online_client | has("credentials_path")')
|
||||||
if isfalse "$config_exists"; then
|
if isfalse "$config_exists"; then
|
||||||
conf_set '.api.server.online_client = {"credentials_path": "/etc/crowdsec/online_api_credentials.yaml"}'
|
export CONFIG_DIR
|
||||||
cscli capi register > /etc/crowdsec/online_api_credentials.yaml
|
conf_set '.api.server.online_client = {"credentials_path": strenv(CONFIG_DIR) + "/online_api_credentials.yaml"}'
|
||||||
|
cscli capi register > "$CONFIG_DIR/online_api_credentials.yaml"
|
||||||
echo "Registration to online API done"
|
echo "Registration to online API done"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
@ -158,84 +241,75 @@ if [ "$GID" != "" ]; then
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# XXX only with LAPI
|
||||||
if istrue "$USE_TLS"; then
|
if istrue "$USE_TLS"; then
|
||||||
agents_allowed_yaml=$(csv2yaml "$AGENTS_ALLOWED_OU") \
|
agents_allowed_yaml=$(csv2yaml "$AGENTS_ALLOWED_OU") \
|
||||||
bouncers_allowed_yaml=$(csv2yaml "$BOUNCERS_ALLOWED_OU") \
|
bouncers_allowed_yaml=$(csv2yaml "$BOUNCERS_ALLOWED_OU") \
|
||||||
conf_set '
|
conf_set_if "$CACERT_FILE" '.api.server.tls.ca_cert_path = strenv(CACERT_FILE)'
|
||||||
.api.server.tls.ca_cert_path = strenv(CACERT_FILE) |
|
conf_set_if "$LAPI_CERT_FILE" '.api.server.tls.cert_file = strenv(LAPI_CERT_FILE)'
|
||||||
.api.server.tls.cert_file = strenv(CERT_FILE) |
|
conf_set_if "$LAPI_KEY_FILE" '.api.server.tls.key_file = strenv(LAPI_KEY_FILE)'
|
||||||
.api.server.tls.key_file = strenv(KEY_FILE) |
|
conf_set_if "$BOUNCERS_ALLOWED_OU" '.api.server.tls.bouncers_allowed_ou = env(bouncers_allowed_yaml)'
|
||||||
.api.server.tls.bouncers_allowed_ou = env(bouncers_allowed_yaml) |
|
conf_set_if "$AGENTS_ALLOWED_OU" '.api.server.tls.agents_allowed_ou = env(agents_allowed_yaml)'
|
||||||
.api.server.tls.agents_allowed_ou = env(agents_allowed_yaml) |
|
else
|
||||||
... comments=""
|
conf_set 'del(.api.server.tls)'
|
||||||
'
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
conf_set ".config_paths.plugin_dir = strenv(PLUGIN_DIR)"
|
conf_set_if "$PLUGIN_DIR" '.config_paths.plugin_dir = strenv(PLUGIN_DIR)'
|
||||||
|
|
||||||
## Install collections, parsers, scenarios & postoverflows
|
## Install collections, parsers, scenarios & postoverflows
|
||||||
cscli hub update
|
cscli hub update
|
||||||
cscli collections upgrade crowdsecurity/linux || true
|
|
||||||
cscli parsers upgrade crowdsecurity/whitelists || true
|
cscli_if_clean collections upgrade crowdsecurity/linux
|
||||||
cscli parsers install crowdsecurity/docker-logs || true
|
cscli_if_clean parsers upgrade crowdsecurity/whitelists
|
||||||
|
cscli_if_clean parsers install crowdsecurity/docker-logs
|
||||||
|
|
||||||
if [ "$COLLECTIONS" != "" ]; then
|
if [ "$COLLECTIONS" != "" ]; then
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
cscli collections install $COLLECTIONS
|
cscli_if_clean collections install "$COLLECTIONS"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ "$PARSERS" != "" ]; then
|
if [ "$PARSERS" != "" ]; then
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
cscli parsers install $PARSERS
|
cscli_if_clean parsers install "$PARSERS"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ "$SCENARIOS" != "" ]; then
|
if [ "$SCENARIOS" != "" ]; then
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
cscli scenarios install $SCENARIOS
|
cscli_if_clean scenarios install "$SCENARIOS"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ "$POSTOVERFLOWS" != "" ]; then
|
if [ "$POSTOVERFLOWS" != "" ]; then
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
cscli postoverflows install $POSTOVERFLOWS
|
cscli_if_clean postoverflows install "$POSTOVERFLOWS"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
## Remove collections, parsers, scenarios & postoverflows
|
## Remove collections, parsers, scenarios & postoverflows
|
||||||
if [ "$DISABLE_COLLECTIONS" != "" ]; then
|
if [ "$DISABLE_COLLECTIONS" != "" ]; then
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
cscli collections remove $DISABLE_COLLECTIONS
|
cscli_if_clean collections remove "$DISABLE_COLLECTIONS"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ "$DISABLE_PARSERS" != "" ]; then
|
if [ "$DISABLE_PARSERS" != "" ]; then
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
cscli parsers remove $DISABLE_PARSERS
|
cscli_if_clean parsers remove "$DISABLE_PARSERS"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ "$DISABLE_SCENARIOS" != "" ]; then
|
if [ "$DISABLE_SCENARIOS" != "" ]; then
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
cscli scenarios remove $DISABLE_SCENARIOS
|
cscli_if_clean scenarios remove "$DISABLE_SCENARIOS"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
|
if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
cscli postoverflows remove $DISABLE_POSTOVERFLOWS
|
cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
register_bouncer() {
|
|
||||||
if ! cscli bouncers list -o json | sed '/^ *"name"/!d;s/^ *"name": "\(.*\)",/\1/' | grep -q "^${NAME}$"; then
|
|
||||||
if cscli bouncers add "${NAME}" -k "${KEY}" > /dev/null; then
|
|
||||||
echo "Registered bouncer for ${NAME}"
|
|
||||||
else
|
|
||||||
echo "Failed to register bouncer for ${NAME}"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
## Register bouncers via env
|
## Register bouncers via env
|
||||||
for BOUNCER in $(compgen -A variable | grep -i BOUNCER_KEY); do
|
for BOUNCER in $(compgen -A variable | grep -i BOUNCER_KEY); do
|
||||||
KEY=$(printf '%s' "${!BOUNCER}")
|
KEY=$(printf '%s' "${!BOUNCER}")
|
||||||
NAME=$(printf '%s' "$BOUNCER" | cut -d_ -f2-)
|
NAME=$(printf '%s' "$BOUNCER" | cut -d_ -f3-)
|
||||||
if [[ -n $KEY ]] && [[ -n $NAME ]]; then
|
if [[ -n $KEY ]] && [[ -n $NAME ]]; then
|
||||||
register_bouncer
|
register_bouncer "$NAME" "$KEY"
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
|
@ -245,7 +319,7 @@ for BOUNCER in /run/secrets/@(bouncer_key|BOUNCER_KEY)* ; do
|
||||||
KEY=$(cat "${BOUNCER}")
|
KEY=$(cat "${BOUNCER}")
|
||||||
NAME=$(echo "${BOUNCER}" | awk -F "/" '{printf $NF}' | cut -d_ -f2-)
|
NAME=$(echo "${BOUNCER}" | awk -F "/" '{printf $NF}' | cut -d_ -f2-)
|
||||||
if [[ -n $KEY ]] && [[ -n $NAME ]]; then
|
if [[ -n $KEY ]] && [[ -n $NAME ]]; then
|
||||||
register_bouncer
|
register_bouncer "$NAME" "$KEY"
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
shopt -u nullglob extglob
|
shopt -u nullglob extglob
|
||||||
|
@ -287,7 +361,7 @@ if istrue "$LEVEL_INFO"; then
|
||||||
ARGS="$ARGS -info"
|
ARGS="$ARGS -info"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
conf_set '.prometheus.listen_port=env(METRICS_PORT)'
|
conf_set_if "$METRICS_PORT" '.prometheus.listen_port=env(METRICS_PORT)'
|
||||||
|
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
exec crowdsec $ARGS
|
exec crowdsec $ARGS
|
||||||
|
|
|
@ -25,6 +25,20 @@ import (
|
||||||
tomb "gopkg.in/tomb.v2"
|
tomb "gopkg.in/tomb.v2"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type DataSourceUnavailableError struct {
|
||||||
|
Name string
|
||||||
|
Err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *DataSourceUnavailableError) Error() string {
|
||||||
|
return fmt.Sprintf("datasource '%s' is not available: %v", e.Name, e.Err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *DataSourceUnavailableError) Unwrap() error {
|
||||||
|
return e.Err
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
// The interface each datasource must implement
|
// The interface each datasource must implement
|
||||||
type DataSource interface {
|
type DataSource interface {
|
||||||
GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module
|
GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module
|
||||||
|
@ -86,8 +100,11 @@ func GetDataSourceIface(dataSourceType string) DataSource {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DataSourceConfigure creates and returns a DataSource object from a configuration,
|
||||||
|
// if the configuration is not valid it returns an error.
|
||||||
|
// If the datasource can't be run (eg. journalctl not available), it still returns an error which
|
||||||
|
// can be checked for the appropriate action.
|
||||||
func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataSource, error) {
|
func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataSource, error) {
|
||||||
|
|
||||||
//we dump it back to []byte, because we want to decode the yaml blob twice :
|
//we dump it back to []byte, because we want to decode the yaml blob twice :
|
||||||
//once to DataSourceCommonCfg, and then later to the dedicated type of the datasource
|
//once to DataSourceCommonCfg, and then later to the dedicated type of the datasource
|
||||||
yamlConfig, err := yaml.Marshal(commonConfig)
|
yamlConfig, err := yaml.Marshal(commonConfig)
|
||||||
|
@ -112,7 +129,7 @@ func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataS
|
||||||
subLogger := clog.WithFields(customLog)
|
subLogger := clog.WithFields(customLog)
|
||||||
/* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */
|
/* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */
|
||||||
if err := dataSrc.CanRun(); err != nil {
|
if err := dataSrc.CanRun(); err != nil {
|
||||||
return nil, errors.Wrapf(err, "datasource %s cannot be run", commonConfig.Source)
|
return nil, &DataSourceUnavailableError{Name: commonConfig.Source, Err: err}
|
||||||
}
|
}
|
||||||
/* configure the actual datasource */
|
/* configure the actual datasource */
|
||||||
if err := dataSrc.Configure(yamlConfig, subLogger); err != nil {
|
if err := dataSrc.Configure(yamlConfig, subLogger); err != nil {
|
||||||
|
@ -179,10 +196,11 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
|
||||||
}
|
}
|
||||||
dec := yaml.NewDecoder(yamlFile)
|
dec := yaml.NewDecoder(yamlFile)
|
||||||
dec.SetStrict(true)
|
dec.SetStrict(true)
|
||||||
|
idx := -1
|
||||||
for {
|
for {
|
||||||
var sub configuration.DataSourceCommonCfg
|
var sub configuration.DataSourceCommonCfg
|
||||||
var idx int
|
|
||||||
err = dec.Decode(&sub)
|
err = dec.Decode(&sub)
|
||||||
|
idx += 1
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if ! errors.Is(err, io.EOF) {
|
if ! errors.Is(err, io.EOF) {
|
||||||
return nil, errors.Wrapf(err, "failed to yaml decode %s", acquisFile)
|
return nil, errors.Wrapf(err, "failed to yaml decode %s", acquisFile)
|
||||||
|
@ -199,7 +217,6 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
|
||||||
if len(sub.Labels) == 0 {
|
if len(sub.Labels) == 0 {
|
||||||
if sub.Source == "" {
|
if sub.Source == "" {
|
||||||
log.Debugf("skipping empty item in %s", acquisFile)
|
log.Debugf("skipping empty item in %s", acquisFile)
|
||||||
idx += 1
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
return nil, fmt.Errorf("missing labels in %s (position: %d)", acquisFile, idx)
|
return nil, fmt.Errorf("missing labels in %s (position: %d)", acquisFile, idx)
|
||||||
|
@ -212,10 +229,14 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
|
||||||
}
|
}
|
||||||
src, err := DataSourceConfigure(sub)
|
src, err := DataSourceConfigure(sub)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
var dserr *DataSourceUnavailableError
|
||||||
|
if errors.As(err, &dserr) {
|
||||||
|
log.Error(err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
return nil, errors.Wrapf(err, "while configuring datasource of type %s from %s (position: %d)", sub.Source, acquisFile, idx)
|
return nil, errors.Wrapf(err, "while configuring datasource of type %s from %s (position: %d)", sub.Source, acquisFile, idx)
|
||||||
}
|
}
|
||||||
sources = append(sources, *src)
|
sources = append(sources, *src)
|
||||||
idx += 1
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return sources, nil
|
return sources, nil
|
||||||
|
|
|
@ -171,7 +171,7 @@ log_level: debug
|
||||||
source: mock_cant_run
|
source: mock_cant_run
|
||||||
wowo: ajsajasjas
|
wowo: ajsajasjas
|
||||||
`,
|
`,
|
||||||
ExpectedError: "datasource mock_cant_run cannot be run: can't run bro",
|
ExpectedError: "datasource 'mock_cant_run' is not available: can't run bro",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -54,7 +54,7 @@ func TestTimestamp(t *testing.T) {
|
||||||
currentYear bool
|
currentYear bool
|
||||||
}{
|
}{
|
||||||
{"May 20 09:33:54", "0000-05-20T09:33:54Z", "", false},
|
{"May 20 09:33:54", "0000-05-20T09:33:54Z", "", false},
|
||||||
{"May 20 09:33:54", "2022-05-20T09:33:54Z", "", true},
|
{"May 20 09:33:54", "2023-05-20T09:33:54Z", "", true},
|
||||||
{"May 20 09:33:54 2022", "2022-05-20T09:33:54Z", "", false},
|
{"May 20 09:33:54 2022", "2022-05-20T09:33:54Z", "", false},
|
||||||
{"May 1 09:33:54 2022", "2022-05-01T09:33:54Z", "", false},
|
{"May 1 09:33:54 2022", "2022-05-01T09:33:54Z", "", false},
|
||||||
{"May 01 09:33:54 2021", "2021-05-01T09:33:54Z", "", true},
|
{"May 01 09:33:54 2021", "2021-05-01T09:33:54Z", "", true},
|
||||||
|
@ -257,7 +257,7 @@ func TestParse(t *testing.T) {
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"<12>May 20 09:33:54 UDMPRO,a2edd0c6ae48,udm-1.10.0.3686 kernel: foo", expected{
|
"<12>May 20 09:33:54 UDMPRO,a2edd0c6ae48,udm-1.10.0.3686 kernel: foo", expected{
|
||||||
Timestamp: time.Date(2022, time.May, 20, 9, 33, 54, 0, time.UTC),
|
Timestamp: time.Date(2023, time.May, 20, 9, 33, 54, 0, time.UTC),
|
||||||
Hostname: "UDMPRO,a2edd0c6ae48,udm-1.10.0.3686",
|
Hostname: "UDMPRO,a2edd0c6ae48,udm-1.10.0.3686",
|
||||||
Tag: "kernel",
|
Tag: "kernel",
|
||||||
PID: "",
|
PID: "",
|
||||||
|
|
|
@ -53,8 +53,8 @@ func NewClient(config *Config) (*ApiClient, error) {
|
||||||
UpdateScenario: config.UpdateScenario,
|
UpdateScenario: config.UpdateScenario,
|
||||||
}
|
}
|
||||||
tlsconfig := tls.Config{InsecureSkipVerify: InsecureSkipVerify}
|
tlsconfig := tls.Config{InsecureSkipVerify: InsecureSkipVerify}
|
||||||
if Cert != nil {
|
|
||||||
tlsconfig.RootCAs = CaCertPool
|
tlsconfig.RootCAs = CaCertPool
|
||||||
|
if Cert != nil {
|
||||||
tlsconfig.Certificates = []tls.Certificate{*Cert}
|
tlsconfig.Certificates = []tls.Certificate{*Cert}
|
||||||
}
|
}
|
||||||
http.DefaultTransport.(*http.Transport).TLSClientConfig = &tlsconfig
|
http.DefaultTransport.(*http.Transport).TLSClientConfig = &tlsconfig
|
||||||
|
@ -75,8 +75,8 @@ func NewDefaultClient(URL *url.URL, prefix string, userAgent string, client *htt
|
||||||
client = &http.Client{}
|
client = &http.Client{}
|
||||||
if ht, ok := http.DefaultTransport.(*http.Transport); ok {
|
if ht, ok := http.DefaultTransport.(*http.Transport); ok {
|
||||||
tlsconfig := tls.Config{InsecureSkipVerify: InsecureSkipVerify}
|
tlsconfig := tls.Config{InsecureSkipVerify: InsecureSkipVerify}
|
||||||
if Cert != nil {
|
|
||||||
tlsconfig.RootCAs = CaCertPool
|
tlsconfig.RootCAs = CaCertPool
|
||||||
|
if Cert != nil {
|
||||||
tlsconfig.Certificates = []tls.Certificate{*Cert}
|
tlsconfig.Certificates = []tls.Certificate{*Cert}
|
||||||
}
|
}
|
||||||
ht.TLSClientConfig = &tlsconfig
|
ht.TLSClientConfig = &tlsconfig
|
||||||
|
|
|
@ -81,7 +81,7 @@ func (l *LocalApiClientCfg) Load() error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if l.Credentials.Login != "" && (l.Credentials.CACertPath != "" || l.Credentials.CertPath != "" || l.Credentials.KeyPath != "") {
|
if l.Credentials.Login != "" && (l.Credentials.CertPath != "" || l.Credentials.KeyPath != "") {
|
||||||
return fmt.Errorf("user/password authentication and TLS authentication are mutually exclusive")
|
return fmt.Errorf("user/password authentication and TLS authentication are mutually exclusive")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -91,12 +91,7 @@ func (l *LocalApiClientCfg) Load() error {
|
||||||
apiclient.InsecureSkipVerify = *l.InsecureSkipVerify
|
apiclient.InsecureSkipVerify = *l.InsecureSkipVerify
|
||||||
}
|
}
|
||||||
|
|
||||||
if l.Credentials.CACertPath != "" && l.Credentials.CertPath != "" && l.Credentials.KeyPath != "" {
|
if l.Credentials.CACertPath != "" {
|
||||||
cert, err := tls.LoadX509KeyPair(l.Credentials.CertPath, l.Credentials.KeyPath)
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrapf(err, "failed to load api client certificate")
|
|
||||||
}
|
|
||||||
|
|
||||||
caCert, err := os.ReadFile(l.Credentials.CACertPath)
|
caCert, err := os.ReadFile(l.Credentials.CACertPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrapf(err, "failed to load cacert")
|
return errors.Wrapf(err, "failed to load cacert")
|
||||||
|
@ -104,10 +99,18 @@ func (l *LocalApiClientCfg) Load() error {
|
||||||
|
|
||||||
caCertPool := x509.NewCertPool()
|
caCertPool := x509.NewCertPool()
|
||||||
caCertPool.AppendCertsFromPEM(caCert)
|
caCertPool.AppendCertsFromPEM(caCert)
|
||||||
|
|
||||||
apiclient.Cert = &cert
|
|
||||||
apiclient.CaCertPool = caCertPool
|
apiclient.CaCertPool = caCertPool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if l.Credentials.CertPath != "" && l.Credentials.KeyPath != "" {
|
||||||
|
cert, err := tls.LoadX509KeyPair(l.Credentials.CertPath, l.Credentials.KeyPath)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "failed to load api client certificate")
|
||||||
|
}
|
||||||
|
|
||||||
|
apiclient.Cert = &cert
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -148,9 +148,10 @@ teardown() {
|
||||||
rm -f "$ACQUIS_DIR"
|
rm -f "$ACQUIS_DIR"
|
||||||
|
|
||||||
config_set '.common.log_media="stdout"'
|
config_set '.common.log_media="stdout"'
|
||||||
run -124 --separate-stderr timeout 2s "${CROWDSEC}"
|
run -1 --separate-stderr timeout 2s "${CROWDSEC}"
|
||||||
# check warning
|
# check warning
|
||||||
assert_stderr_line --partial "no acquisition file found"
|
assert_stderr --partial "no acquisition file found"
|
||||||
|
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "crowdsec (error if acquisition_path and acquisition_dir are not defined)" {
|
@test "crowdsec (error if acquisition_path and acquisition_dir are not defined)" {
|
||||||
|
@ -163,19 +164,55 @@ teardown() {
|
||||||
config_set '.crowdsec_service.acquisition_dir=""'
|
config_set '.crowdsec_service.acquisition_dir=""'
|
||||||
|
|
||||||
config_set '.common.log_media="stdout"'
|
config_set '.common.log_media="stdout"'
|
||||||
run -124 --separate-stderr timeout 2s "${CROWDSEC}"
|
run -1 --separate-stderr timeout 2s "${CROWDSEC}"
|
||||||
# check warning
|
# check warning
|
||||||
assert_stderr_line --partial "no acquisition_path or acquisition_dir specified"
|
assert_stderr --partial "no acquisition_path or acquisition_dir specified"
|
||||||
|
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" {
|
@test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" {
|
||||||
ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
|
ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
|
||||||
rm -f "$ACQUIS_YAML"
|
|
||||||
config_set '.crowdsec_service.acquisition_path=""'
|
config_set '.crowdsec_service.acquisition_path=""'
|
||||||
|
|
||||||
ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir')
|
ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir')
|
||||||
mkdir -p "$ACQUIS_DIR"
|
mkdir -p "$ACQUIS_DIR"
|
||||||
touch "$ACQUIS_DIR"/foo.yaml
|
mv "$ACQUIS_YAML" "$ACQUIS_DIR"/foo.yaml
|
||||||
|
|
||||||
run -124 --separate-stderr timeout 2s "${CROWDSEC}"
|
run -124 --separate-stderr timeout 2s "${CROWDSEC}"
|
||||||
|
|
||||||
|
# now, if foo.yaml is empty instead, there won't be valid datasources.
|
||||||
|
|
||||||
|
cat /dev/null >"$ACQUIS_DIR"/foo.yaml
|
||||||
|
|
||||||
|
run --separate-stderr -1 timeout 2s "${CROWDSEC}"
|
||||||
|
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "crowdsec (disabled datasources)" {
|
||||||
|
config_set '.common.log_media="stdout"'
|
||||||
|
|
||||||
|
# a datasource cannot run - missing journalctl command
|
||||||
|
|
||||||
|
ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir')
|
||||||
|
mkdir -p "$ACQUIS_DIR"
|
||||||
|
cat >"$ACQUIS_DIR"/foo.yaml <<-EOT
|
||||||
|
source: journalctl
|
||||||
|
journalctl_filter:
|
||||||
|
- "_SYSTEMD_UNIT=ssh.service"
|
||||||
|
labels:
|
||||||
|
type: syslog
|
||||||
|
EOT
|
||||||
|
|
||||||
|
run --separate-stderr -124 timeout 2s env PATH='' "${CROWDSEC}"
|
||||||
|
#shellcheck disable=SC2016
|
||||||
|
assert_stderr --partial 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in $PATH'
|
||||||
|
|
||||||
|
# if all datasources are disabled, crowdsec should exit
|
||||||
|
|
||||||
|
ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
|
||||||
|
rm -f "$ACQUIS_YAML"
|
||||||
|
config_set '.crowdsec_service.acquisition_path=""'
|
||||||
|
|
||||||
|
run --separate-stderr -1 timeout 2s env PATH='' "${CROWDSEC}"
|
||||||
|
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
is_crowdsec_running() {
|
is_crowdsec_running() {
|
||||||
PIDS=$(pgrep -x 'crowdsec|crowdsec.test|crowdsec.cover')
|
PIDS=$(pgrep -x 'crowdsec|crowdsec.test|crowdsec.cover' 2>/dev/null)
|
||||||
}
|
}
|
||||||
|
|
||||||
# The process can be slow, especially on CI and during test coverage.
|
# The process can be slow, especially on CI and during test coverage.
|
||||||
|
|
Loading…
Reference in a new issue