Compare commits
4 commits
master
...
docker_con
Author | SHA1 | Date | |
---|---|---|---|
|
3ce82cb400 | ||
|
a895b16fc7 | ||
|
eeaf508a09 | ||
|
f81bb0a3d9 |
14
.github/workflows/docker-tests.yml
vendored
14
.github/workflows/docker-tests.yml
vendored
|
@ -59,15 +59,15 @@ jobs:
|
||||||
cd docker/test
|
cd docker/test
|
||||||
python -m pip install --upgrade pipenv wheel
|
python -m pip install --upgrade pipenv wheel
|
||||||
|
|
||||||
#- name: "Cache virtualenvs"
|
- name: "Cache virtualenvs"
|
||||||
# id: cache-pipenv
|
id: cache-pipenv
|
||||||
# uses: actions/cache@v4
|
uses: actions/cache@v4
|
||||||
# with:
|
with:
|
||||||
# path: ~/.local/share/virtualenvs
|
path: ~/.local/share/virtualenvs
|
||||||
# key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }}
|
key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }}
|
||||||
|
|
||||||
- name: "Install dependencies"
|
- name: "Install dependencies"
|
||||||
#if: steps.cache-pipenv.outputs.cache-hit != 'true'
|
if: steps.cache-pipenv.outputs.cache-hit != 'true'
|
||||||
run: |
|
run: |
|
||||||
cd docker/test
|
cd docker/test
|
||||||
pipenv install --deploy
|
pipenv install --deploy
|
||||||
|
|
|
@ -25,6 +25,7 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
|
||||||
./wizard.sh --docker-mode && \
|
./wizard.sh --docker-mode && \
|
||||||
cd - >/dev/null && \
|
cd - >/dev/null && \
|
||||||
cscli hub update && \
|
cscli hub update && \
|
||||||
|
./docker/preload-hub-items && \
|
||||||
cscli collections install crowdsecurity/linux && \
|
cscli collections install crowdsecurity/linux && \
|
||||||
cscli parsers install crowdsecurity/whitelists
|
cscli parsers install crowdsecurity/whitelists
|
||||||
|
|
||||||
|
|
|
@ -30,6 +30,7 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
|
||||||
./wizard.sh --docker-mode && \
|
./wizard.sh --docker-mode && \
|
||||||
cd - >/dev/null && \
|
cd - >/dev/null && \
|
||||||
cscli hub update && \
|
cscli hub update && \
|
||||||
|
./docker/preload-hub-items && \
|
||||||
cscli collections install crowdsecurity/linux && \
|
cscli collections install crowdsecurity/linux && \
|
||||||
cscli parsers install crowdsecurity/whitelists
|
cscli parsers install crowdsecurity/whitelists
|
||||||
|
|
||||||
|
|
|
@ -50,6 +50,54 @@ cscli() {
|
||||||
command cscli -c "$CONFIG_FILE" "$@"
|
command cscli -c "$CONFIG_FILE" "$@"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
run_hub_update() {
|
||||||
|
index_modification_time=$(stat -c %Y /etc/crowdsec/hub/.index.json 2>/dev/null)
|
||||||
|
#Run cscli hub update if no date or if the index file is older than 24h
|
||||||
|
if [ -z "$index_modification_time" ] || [ $(( $(date +%s) - $index_modification_time )) -gt 86400 ]; then
|
||||||
|
cscli hub update
|
||||||
|
else
|
||||||
|
echo "Skipping hub update, index file is recent"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
is_mounted() {
|
||||||
|
path=$(readlink -f "$1")
|
||||||
|
mounts=$(awk '{print $2}' /proc/mounts)
|
||||||
|
while true; do
|
||||||
|
if grep -qE ^"$path"$ <<< "$mounts"; then
|
||||||
|
echo "$path was found in a volume"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
path=$(dirname "$path")
|
||||||
|
if [ "$path" = "/" ]; then
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
return 1 #unreachable
|
||||||
|
}
|
||||||
|
|
||||||
|
run_hub_update_if_from_volume() {
|
||||||
|
is_mounted "/etc/crowdsec/hub/.index.json"
|
||||||
|
if [ $? -eq 0 ]; then
|
||||||
|
echo "Running hub update"
|
||||||
|
run_hub_update
|
||||||
|
else
|
||||||
|
echo "Skipping hub update, index file is not in a volume"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
run_hub_upgrade_if_from_volume() {
|
||||||
|
isfalse "$NO_HUB_UPGRADE" || return 0
|
||||||
|
is_mounted "/var/lib/crowdsec/data"
|
||||||
|
if [ $? -eq 0 ]; then
|
||||||
|
echo "Running hub upgrade"
|
||||||
|
cscli hub upgrade
|
||||||
|
else
|
||||||
|
echo "Skipping hub upgrade, data directory is not in a volume"
|
||||||
|
fi
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
# conf_get <key> [file_path]
|
# conf_get <key> [file_path]
|
||||||
# retrieve a value from a file (by default $CONFIG_FILE)
|
# retrieve a value from a file (by default $CONFIG_FILE)
|
||||||
conf_get() {
|
conf_get() {
|
||||||
|
@ -120,6 +168,12 @@ cscli_if_clean() {
|
||||||
echo "Running: cscli $error_only $itemtype $action \"$obj\" $*"
|
echo "Running: cscli $error_only $itemtype $action \"$obj\" $*"
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
cscli $error_only "$itemtype" "$action" "$obj" "$@"
|
cscli $error_only "$itemtype" "$action" "$obj" "$@"
|
||||||
|
if [ "$?" -ne 0 ]; then
|
||||||
|
echo "Failed to $action $itemtype/$obj, running hub update before retrying"
|
||||||
|
run_hub_update
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
cscli $error_only "$itemtype" "$action" "$obj" "$@"
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
@ -304,11 +358,8 @@ conf_set_if "$PLUGIN_DIR" '.config_paths.plugin_dir = strenv(PLUGIN_DIR)'
|
||||||
|
|
||||||
## Install hub items
|
## Install hub items
|
||||||
|
|
||||||
cscli hub update || true
|
run_hub_update_if_from_volume || true
|
||||||
|
run_hub_upgrade_if_from_volume || true
|
||||||
if isfalse "$NO_HUB_UPGRADE"; then
|
|
||||||
cscli hub upgrade || true
|
|
||||||
fi
|
|
||||||
|
|
||||||
cscli_if_clean parsers install crowdsecurity/docker-logs
|
cscli_if_clean parsers install crowdsecurity/docker-logs
|
||||||
cscli_if_clean parsers install crowdsecurity/cri-logs
|
cscli_if_clean parsers install crowdsecurity/cri-logs
|
||||||
|
|
22
docker/preload-hub-items
Executable file
22
docker/preload-hub-items
Executable file
|
@ -0,0 +1,22 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
# pre-download everything but don't install anything
|
||||||
|
|
||||||
|
echo "Pre-downloading Hub content..."
|
||||||
|
|
||||||
|
types=$(cscli hub types -o raw)
|
||||||
|
|
||||||
|
for itemtype in $types; do
|
||||||
|
ALL_ITEMS=$(cscli "$itemtype" list -a -o json | itemtype="$itemtype" yq '.[env(itemtype)][] | .name')
|
||||||
|
if [[ -n "${ALL_ITEMS}" ]]; then
|
||||||
|
#shellcheck disable=SC2086
|
||||||
|
cscli "$itemtype" install \
|
||||||
|
$ALL_ITEMS \
|
||||||
|
--download-only \
|
||||||
|
--error
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
echo " done."
|
|
@ -9,20 +9,12 @@ THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||||
|
|
||||||
# pre-download everything but don't install anything
|
# pre-download everything but don't install anything
|
||||||
|
|
||||||
echo -n "Purging existing hub..."
|
echo "Pre-downloading Hub content..."
|
||||||
|
|
||||||
types=$("$CSCLI" hub types -o raw)
|
types=$("$CSCLI" hub types -o raw)
|
||||||
|
|
||||||
for itemtype in $types; do
|
for itemtype in $types; do
|
||||||
"$CSCLI" "${itemtype}" delete --all --error --purge --force
|
ALL_ITEMS=$("$CSCLI" "$itemtype" list -a -o json | itemtype="$itemtype" yq '.[env(itemtype)][] | .name')
|
||||||
done
|
|
||||||
|
|
||||||
echo " done."
|
|
||||||
|
|
||||||
echo -n "Pre-downloading Hub content..."
|
|
||||||
|
|
||||||
for itemtype in $types; do
|
|
||||||
ALL_ITEMS=$("$CSCLI" "$itemtype" list -a -o json | jq --arg itemtype "$itemtype" -r '.[$itemtype][].name')
|
|
||||||
if [[ -n "${ALL_ITEMS}" ]]; then
|
if [[ -n "${ALL_ITEMS}" ]]; then
|
||||||
#shellcheck disable=SC2086
|
#shellcheck disable=SC2086
|
||||||
"$CSCLI" "$itemtype" install \
|
"$CSCLI" "$itemtype" install \
|
||||||
|
@ -32,11 +24,4 @@ for itemtype in $types; do
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
# XXX: download-only works only for collections, not for parsers, scenarios, postoverflows.
|
|
||||||
# so we have to delete the links manually, and leave the downloaded files in place
|
|
||||||
|
|
||||||
for itemtype in $types; do
|
|
||||||
"$CSCLI" "$itemtype" delete --all --error
|
|
||||||
done
|
|
||||||
|
|
||||||
echo " done."
|
echo " done."
|
Loading…
Reference in a new issue