2020-05-15 09:39:16 +00:00
#!/usr/bin/env bash
set -o pipefail
#set -x
RED = '\033[0;31m'
BLUE = '\033[0;34m'
GREEN = '\033[0;32m'
YELLOW = '\033[0;33m'
NC = '\033[0m'
SILENT = "false"
CROWDSEC_RUN_DIR = "/var/run"
CROWDSEC_LIB_DIR = "/var/lib/crowdsec"
CROWDSEC_DATA_DIR = " ${ CROWDSEC_LIB_DIR } /data "
CROWDSEC_PLUGIN_DIR = " ${ CROWDSEC_LIB_DIR } /plugins "
CROWDSEC_PLUGIN_BACKEND_DIR = " ${ CROWDSEC_PLUGIN_DIR } /backend "
CROWDSEC_DB_PATH = " ${ CROWDSEC_DATA_DIR } /crowdsec.db "
CROWDSEC_CONFIG_PATH = "/etc/crowdsec"
2020-05-15 10:25:34 +00:00
CROWDSEC_CONFIG_PATH = " ${ CROWDSEC_CONFIG_PATH } /config "
2020-05-15 09:39:16 +00:00
CROWDSEC_LOG_FILE = "/var/log/crowdsec.log"
CROWDSEC_BACKEND_FOLDER = "/etc/crowdsec/plugins/backend"
2020-05-24 16:40:00 +00:00
CSCLI_FOLDER = "/etc/crowdsec/config/cscli"
2020-05-15 09:39:16 +00:00
CROWDSEC_BIN = "./cmd/crowdsec/crowdsec"
CSCLI_BIN = "./cmd/crowdsec-cli/cscli"
CROWDSEC_BIN_INSTALLED = "/usr/local/bin/crowdsec"
CSCLI_BIN_INSTALLED = "/usr/local/bin/cscli"
ACQUIS_PATH = " ${ CROWDSEC_CONFIG_PATH } "
TMP_ACQUIS_FILE = "tmp-acquis.yaml"
ACQUIS_TARGET = " ${ ACQUIS_PATH } /acquis.yaml "
setup_cron_pull( ) {
cp ./config/crowdsec_pull /etc/cron.d/
}
PID_DIR = "/var/run"
SYSTEMD_PATH_FILE = "/etc/systemd/system/crowdsec.service"
PATTERNS_FOLDER = "config/patterns"
PATTERNS_PATH = " ${ CROWDSEC_CONFIG_PATH } /patterns/ "
ACTION = ""
DEBUG_MODE = "false"
SUPPORTED_SERVICES = ' apache2
nginx
sshd
mysql
telnet
smb
'
BACKUP_DIR = $( mktemp -d)
log_info( ) {
msg = $1
date = $( date +%x:%X)
echo -e " [ ${ date } ][ ${ BLUE } INF ${ NC } ] crowdsec_wizard: ${ msg } "
}
log_err( ) {
msg = $1
date = $( date +%x:%X)
echo -e " [ ${ date } ][ ${ RED } ERR ${ NC } ] crowdsec_wizard: ${ msg } " 1>& 2
}
log_dbg( ) {
if [ [ ${ DEBUG_MODE } = = "true" ] ] ; then
msg = $1
date = $( date +%x:%X)
echo -e " [ ${ date } ][ ${ YELLOW } DBG ${ NC } ] crowdsec_wizard: ${ msg } " 1>& 2
fi
}
detect_services ( ) {
DETECTED_SERVICES = ( )
HMENU = ( )
#list systemd services
SYSTEMD_SERVICES = ` systemctl --state= enabled list-unit-files '*.service' | cut -d ' ' -f1`
#raw ps
PSAX = ` ps ax -o comm = `
for SVC in ${ SUPPORTED_SERVICES } ; do
log_info " Checking if service ' ${ SVC } ' is running (ps+systemd) "
for SRC in " ${ SYSTEMD_SERVICES } " " ${ PSAX } " ; do
echo ${ SRC } | grep ${ SVC } >/dev/null
if [ $? -eq 0 ] ; then
DETECTED_SERVICES += ( ${ SVC } )
HMENU += ( ${ SVC } "on" )
log_info " Found ' ${ SVC } ' running "
break;
fi ;
done ;
done ;
if [ [ ${ OSTYPE } = = "linux-gnu" ] ] ; then
DETECTED_SERVICES += ( "linux" )
HMENU += ( "linux" "on" )
else
log_info "NOT A LINUX"
fi ;
if [ [ ${ SILENT } = = "false" ] ] ; then
#we put whiptail results in an array, notice the dark magic fd redirection
DETECTED_SERVICES = ( $( whiptail --separate-output --noitem --ok-button Continue --title "Services to monitor" --checklist "Detected services, uncheck to ignore. Ignored services won't be monitored." 18 70 10 ${ HMENU [@] } 3>& 1 1>& 2 2>& 3) )
if [ $? -eq 1 ] ; then
log_err "user bailed out at services selection"
exit 1;
fi ;
fi ;
}
declare -A log_input_tags
log_input_tags[ apache2] = 'type: apache2'
log_input_tags[ nginx] = 'type: nginx'
log_input_tags[ sshd] = 'type: syslog'
log_input_tags[ rsyslog] = 'type: syslog'
log_input_tags[ telnet] = 'type: telnet'
log_input_tags[ mysql] = 'type: mysql'
log_input_tags[ smb] = 'type: smb'
log_input_tags[ linux] = "type: syslog"
declare -A log_locations
log_locations[ apache2] = '/var/log/apache2/*.log,/var/log/*httpd*.log'
log_locations[ nginx] = '/var/log/nginx/*.log'
log_locations[ sshd] = '/var/log/auth.log,/var/log/sshd.log,/var/log/secure'
log_locations[ rsyslog] = '/var/log/syslog'
log_locations[ telnet] = '/var/log/telnetd*.log'
log_locations[ mysql] = '/var/log/mysqld*.log'
log_locations[ smb] = '/var/log/samba*.log'
log_locations[ linux] = '/var/log/syslog,/var/log/kern.log,/var/log/messages'
#$1 is service name, such those in SUPPORTED_SERVICES
find_logs_for( ) {
ret = ""
x = ${ 1 }
#we have trailing and starting quotes because of whiptail
SVC = " ${ x % \" } "
SVC = " ${ SVC # \" } "
DETECTED_LOGFILES = ( )
HMENU = ( )
#log_info "Searching logs for ${SVC} : ${log_locations[${SVC}]}"
#split the line into an array with ',' separator
OIFS = ${ IFS }
IFS = ',' read -r -a a <<< " ${ log_locations [ ${ SVC } ] } , "
IFS = ${ OIFS }
#readarray -td, a <<<"${log_locations[${SVC}]},"; unset 'a[-1]';
for poss_path in " ${ a [@] } " ; do
#Split /var/log/nginx/*.log into '/var/log/nginx' and '*.log' so we can use find
path = ${ poss_path %/* }
fname = ${ poss_path ##*/ }
candidates = ` find " ${ path } " -type f -mtime -5 -ctime -5 -name " $fname " `
#We have some candidates, add them
for final_file in ${ candidates } ; do
log_info " Found logs file for ' ${ SVC } ': ${ final_file } "
DETECTED_LOGFILES += ( ${ final_file } )
HMENU += ( ${ final_file } "on" )
done ;
done ;
if [ [ ${ SILENT } = = "false" ] ] ; then
DETECTED_LOGFILES = ( $( whiptail --separate-output --noitem --ok-button Continue --title " Log files to process for ${ SVC } " --checklist " Detected logfiles for ${ SVC } , uncheck to ignore " 18 70 10 ${ HMENU [@] } 3>& 1 1>& 2 2>& 3) )
if [ $? -eq 1 ] ; then
log_err "user bailed out at log file selection"
exit 1;
fi ;
fi
}
in_array( ) {
str = $1
shift
array = ( " $@ " )
for element in " ${ array [@] } " ; do
if [ [ ${ str } = = *${ element } * ] ] ; then
return 0
fi
done
return 1
}
install_collection( ) {
HMENU = ( )
readarray -t AVAILABLE_COLLECTION < <( ${ CSCLI_BIN_INSTALLED } list collections -o raw -a)
COLLECTION_TO_INSTALL = ( )
if [ [ ${ SILENT } = = "false" ] ] ; then
for collect_info in " ${ AVAILABLE_COLLECTION [@] } " ; do
#echo "collection raw : ${collect_info}" >> out.txt
collection = " $( echo ${ collect_info } | cut -d " " -f1) "
description = " $( echo ${ collect_info } | cut -d " " -f2-) "
in_array $collection " ${ DETECTED_SERVICES [@] } "
if [ [ $? = = 0 ] ] ; then
HMENU += ( " ${ collection } " " ${ description } " "ON" )
else
if [ [ ${ collection } = = "linux" ] ] ; then
HMENU += ( " ${ collection } " " ${ description } " "ON" )
else
HMENU += ( " ${ collection } " " ${ description } " "OFF" )
fi
fi
done
COLLECTION_TO_INSTALL = ( $( whiptail --separate-output --ok-button Continue --title "Crowdsec collections" --checklist "Available collections in crowdsec, try to pick one that fits your profile. Collections contains parsers and scenarios to protect your system." 20 120 10 " ${ HMENU [@] } " 3>& 1 1>& 2 2>& 3) )
if [ $? -eq 1 ] ; then
log_err "user bailed out at collection selection"
exit 1;
fi ;
else
for collection in " ${ DETECTED_SERVICES [@] } " ; do
COLLECTION_TO_INSTALL += ( ${ collection } )
done
fi
for collection in " ${ COLLECTION_TO_INSTALL [@] } " ; do
log_info " Installing collection ' ${ collection } ' "
${ CSCLI_BIN_INSTALLED } install collection " ${ collection } " > /dev/null 2>& 1 || log_err " fail to install collection ${ collection } "
done
${ CSCLI_BIN_INSTALLED } install parser "crowdsecurity/whitelists" > /dev/null 2>& 1 || log_err "fail to install collection crowdsec/whitelists"
if [ [ ${ SILENT } = = "false" ] ] ; then
whiptail --msgbox "Out of safety, I installed a parser called 'crowdsecurity/whitelists'. This one will prevent private IP adresses from being banned, feel free to remove it any time." 20 50
fi
}
#$1 is the service name, $... is the list of candidate logs (from find_logs_for)
genyaml( ) {
local service = " ${ 1 } "
shift
local files = ( " ${ @ } " )
echo " #Generated acquisition file - wizard.sh (service: ${ service } ) / files : ${ files [@] } " >> ${ TMP_ACQUIS_FILE }
echo "filenames:" >> ${ TMP_ACQUIS_FILE }
for fd in ${ files [@] } ; do
echo " - ${ fd } " >> ${ TMP_ACQUIS_FILE }
done
echo "labels:" >> ${ TMP_ACQUIS_FILE }
echo " " ${ log_input_tags [ ${ service } ] } >> ${ TMP_ACQUIS_FILE }
echo "---" >> ${ TMP_ACQUIS_FILE }
log_info "Acquisition file generated"
}
genacquisition( ) {
log_info "Found following services : " ${ DETECTED_SERVICES [@] }
for PSVG in ${ DETECTED_SERVICES [@] } ; do
find_logs_for ${ PSVG }
if [ [ ${# DETECTED_LOGFILES [@] } -gt 0 ] ] ; then
genyaml ${ PSVG } ${ DETECTED_LOGFILES [@] }
fi ;
done
}
#install crowdsec and cscli
install_crowdsec( ) {
mkdir -p " ${ CROWDSEC_DATA_DIR } "
find data -type f -exec install -Dm 755 { } " ${ CROWDSEC_LIB_DIR } /{} " \; || exit
( cd config && find patterns -type f -exec install -Dm 755 "{}" " ${ CROWDSEC_CONFIG_PATH } /{} " \; && cd ../) || exit
mkdir -p " ${ CROWDSEC_CONFIG_PATH } /scenarios " || exit
mkdir -p " ${ CROWDSEC_CONFIG_PATH } /postoverflows " || exit
mkdir -p " ${ CROWDSEC_CONFIG_PATH } /collections " || exit
mkdir -p " ${ CROWDSEC_CONFIG_PATH } /patterns " || exit
mkdir -p " ${ CROWDSEC_BACKEND_FOLDER } " || exit
2020-05-24 16:40:00 +00:00
mkdir -p " ${ CROWDSEC_PLUGIN_BACKEND_DIR } " || exit
mkdir -p " ${ CSCLI_FOLDER } " || exit
2020-05-15 09:39:16 +00:00
( cd ./plugins && find . -type f -name "*.so" -exec install -Dm 644 { } " ${ CROWDSEC_PLUGIN_DIR } /{} " \; && cd ../) || exit
cp -r ./config/plugins/backend/* " ${ CROWDSEC_BACKEND_FOLDER } " || exit
install -v -m 755 -D ./config/prod.yaml " ${ CROWDSEC_CONFIG_PATH } " || exit
install -v -m 755 -D ./config/dev.yaml " ${ CROWDSEC_CONFIG_PATH } " || exit
install -v -m 755 -D ./config/acquis.yaml " ${ CROWDSEC_CONFIG_PATH } " || exit
install -v -m 755 -D ./config/profiles.yaml " ${ CROWDSEC_CONFIG_PATH } " || exit
install -v -m 600 -D ./config/api.yaml " ${ CROWDSEC_CONFIG_PATH } " || exit
mkdir -p ${ PID_DIR } || exit
PID = ${ PID_DIR } DATA = ${ CROWDSEC_DATA_DIR } CFG = ${ CROWDSEC_CONFIG_PATH } envsubst < ./config/prod.yaml > ${ CROWDSEC_CONFIG_PATH } "/default.yaml"
2020-05-26 13:41:40 +00:00
PID = ${ PID_DIR } DATA = ${ CROWDSEC_DATA_DIR } CFG = ${ CROWDSEC_CONFIG_PATH } envsubst < ./config/user.yaml > ${ CROWDSEC_CONFIG_PATH } "/user.yaml"
2020-05-15 09:39:16 +00:00
CFG = ${ CROWDSEC_CONFIG_PATH } PID = ${ PID_DIR } BIN = ${ CROWDSEC_BIN_INSTALLED } envsubst < ./config/crowdsec.service > " ${ SYSTEMD_PATH_FILE } "
install_bins
systemctl daemon-reload
}
update_bins( ) {
log_info "Saving api credentials"
${ CSCLI_BIN_INSTALLED } api credentials > ${ BACKUP_DIR } /api-credentials.back || log_err "Unable to save credentials"
cat ${ CROWDSEC_CONFIG_PATH } /api.yaml
log_info "Only upgrading binaries"
delete_bins
install_bins
log_info "Upgrade finished"
echo "#restored credentials" >> ${ CROWDSEC_CONFIG_PATH } /api.yaml || log_err "unable to write to api file"
cat ${ BACKUP_DIR } /api-credentials.back >> ${ CROWDSEC_CONFIG_PATH } /api.yaml || log_err "unable to write to api file"
systemctl restart crowdsec
}
update_full( ) {
if [ [ ! -f " $CROWDSEC_BIN " ] ] ; then
log_err " Crowdwatch binary ' $CROWDSEC_BIN ' not found. Please build it with 'make build' " && exit
fi
if [ [ ! -f " $CSCLI_BIN " ] ] ; then
log_err " Cwcli binary ' $CSCLI_BIN ' not found. Please build it with 'make build' " && exit
fi
log_info "Backing up existing configuration"
${ CSCLI_BIN } backup save ${ BACKUP_DIR }
log_info "Cleanup existing crowdsec configuration"
uninstall_crowdsec
log_info "Installing crowdsec"
install_crowdsec
log_info "Restoring configuration"
${ CSCLI_BIN } backup restore ${ BACKUP_DIR }
log_info "Finished, restarting"
systemctl restart crowdsec || log_err "Failed to restart crowdsec"
}
install_bins( ) {
log_info "Installing crowdsec binaries"
install -v -m 755 -D " ${ CROWDSEC_BIN } " " ${ CROWDSEC_BIN_INSTALLED } " || exit
install -v -m 755 -D " ${ CSCLI_BIN } " " ${ CSCLI_BIN_INSTALLED } " || exit
}
delete_bins( ) {
log_info "Removing crowdsec binaries"
rm -f ${ CROWDSEC_BIN_INSTALLED }
rm -f ${ CSCLI_BIN_INSTALLED }
}
# uninstall crowdsec and cscli
uninstall_crowdsec( ) {
systemctl stop crowdsec.service
${ CSCLI_BIN } dashboard stop --remove
delete_bins
rm -rf ${ CROWDSEC_CONFIG_PATH } || echo ""
rm -f ${ CROWDSEC_LOG_FILE } || echo ""
rm -f ${ CROWDSEC_DB_PATH } || echo ""
rm -rf ${ CROWDSEC_LIB_DIR } || echo ""
rm -f ${ SYSTEMD_PATH_FILE } || echo ""
log_info "crowdsec successfully uninstalled"
}
setup_cron_pull( ) {
cp ./config/crowdsec_pull /etc/cron.d/
}
main( ) {
if [ [ " $1 " = = "backup_to_dir" ] ] ;
then
backup_to_dir
return
fi
if [ [ " $1 " = = "restore_from_dir" ] ] ;
then
if ! [ $( id -u) = 0 ] ; then
log_err "Please run it as root"
exit 1
fi
restore_from_dir
return
fi
if [ [ " $1 " = = "binupgrade" ] ] ;
then
if ! [ $( id -u) = 0 ] ; then
log_err "Please run it as root"
exit 1
fi
update_bins
return
fi
if [ [ " $1 " = = "upgrade" ] ] ;
then
if ! [ $( id -u) = 0 ] ; then
log_err "Please run it as root"
exit 1
fi
update_full
return
fi
if [ [ " $1 " = = "uninstall" ] ] ;
then
if ! [ $( id -u) = 0 ] ; then
log_err "Please run it as root"
exit 1
fi
uninstall_crowdsec
return
fi
if [ [ " $1 " = = "bininstall" ] ] ;
then
if ! [ $( id -u) = 0 ] ; then
log_err "Please run it as root"
exit 1
fi
log_info "installing crowdsec"
install_crowdsec
return
fi
if [ [ " $1 " = = "install" ] ] ;
then
if ! [ $( id -u) = 0 ] ; then
log_err "Please run it as root"
exit 1
fi
## Do make build before installing (as non--root) in order to have the binary and then install crowdsec as root
log_info "installing crowdsec"
install_crowdsec
log_info " configuring ${ CSCLI_BIN_INSTALLED } "
${ CSCLI_BIN_INSTALLED } update > /dev/null 2>& 1 || ( log_err "fail to update crowdsec hub. exiting" && exit 1)
# detect running services
detect_services
if ! [ ${# DETECTED_SERVICES [@] } -gt 0 ] ; then
log_err "No detected or selected services, stopping."
exit 1
fi ;
# Generate acquisition file and move it to the right folder
genacquisition
mv " ${ TMP_ACQUIS_FILE } " " ${ ACQUIS_TARGET } "
# Install collections according to detected services
log_info "Installing needed collections ..."
install_collection
# install patterns/ folder
log_info "Installing patterns"
mkdir -p " ${ PATTERNS_PATH } "
cp " ./ ${ PATTERNS_FOLDER } / " * " ${ PATTERNS_PATH } / "
# api register
2020-05-15 10:13:57 +00:00
${ CSCLI_BIN_INSTALLED } api register >> /etc/crowdsec/config/api.yaml || ${ CSCLI_BIN_INSTALLED } api reset >> /etc/crowdsec/config/api.yaml || log_err "unable to register, skipping crowdsec api registration"
2020-05-15 09:39:16 +00:00
log_info "Crowdsec api registered"
( systemctl start crowdsec && log_info "crowdsec started" ) || log_err "unable to start crowdsec. exiting"
${ CSCLI_BIN_INSTALLED } api pull
# Set the cscli api pull cronjob
setup_cron_pull
return
fi
if [ [ " $1 " = = "detect" ] ] ;
then
rm -f " ${ TMP_ACQUIS_FILE } "
detect_services
if [ [ ${ DETECTED_SERVICES } = = "" ] ] ; then
log_err "No detected or selected services, stopping."
exit
fi ;
log_info " Found ${# DETECTED_SERVICES [@] } supported services running: "
genacquisition
cat " ${ TMP_ACQUIS_FILE } "
rm " ${ TMP_ACQUIS_FILE } "
return
fi
}
usage( ) {
echo "Usage:"
echo " ./wizard.sh -h Display this help message."
echo " ./wizard.sh -d|--detect Detect running services and associated logs file"
echo " ./wizard.sh -i|--install Assisted installation of crowdsec/cscli and collections"
echo " ./wizard.sh --bininstall Install binaries and empty config, no wizard."
echo " ./wizard.sh --uninstall Uninstall crowdsec/cscli"
echo " ./wizard.sh --binupgrade Upgrade crowdsec/cscli binaries"
echo " ./wizard.sh --upgrade Perform a full upgrade and try to migrate configs"
echo " ./wizard.sh --unattended Install in unattended mode, no question will be asked and defaults will be followed"
echo " ./wizard.sh -r|--restore Restore saved configurations from ${ BACKUP_DIR } to ${ CROWDSEC_CONFIG_PATH } "
echo " ./wizard.sh -b|--backup Backup existing configurations to ${ BACKUP_DIR } "
exit 0
}
if [ [ $# -eq 0 ] ] ; then
usage
fi
while [ [ $# -gt 0 ] ]
do
key = " ${ 1 } "
case ${ key } in
--uninstall)
ACTION = "uninstall"
shift #past argument
; ;
--binupgrade)
ACTION = "binupgrade"
shift #past argument
; ;
--upgrade)
ACTION = "upgrade"
shift #past argument
; ;
-i| --install)
ACTION = "install"
shift # past argument
; ;
--bininstall)
ACTION = "bininstall"
shift # past argument
; ;
-b| --backup)
ACTION = "backup_to_dir"
shift # past argument
; ;
-r| --restore)
ACTION = "restore_from_dir"
shift # past argument
; ;
-d| --detect)
ACTION = "detect"
shift # past argument
; ;
--unattended)
SILENT = "true"
ACTION = "install"
shift
; ;
-v| --verbose)
DEBUG_MODE = "true"
shift
; ;
-h| --help)
usage
exit 0
; ;
*) # unknown option
log_err " Unknown argument ${ key } . "
usage
exit 1
; ;
esac
done
main ${ ACTION }