beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

improve logging in cscli and wizard (#643)

Browse source
This commit is contained in:
AlteredCoder 2021-02-25 11:20:36 +01:00 committed by GitHub
parent a3d00fe130
commit 8b504e9f67
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 88 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cmd/crowdsec-cli/capi.go
View file

@ -91,7 +91,7 @@ func NewCapiCmd() *cobra.Command {
if err != nil { if err != nil {
log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err) log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
} }
log.Printf("API credentials dumped to '%s'", dumpFile) log.Printf("Central API credentials dumped to '%s'", dumpFile)
} else { } else {
fmt.Printf("%s\n", string(apiConfigDump)) fmt.Printf("%s\n", string(apiConfigDump))
} }

6
cmd/crowdsec-cli/dashboard.go
View file

@ -7,6 +7,7 @@ import (
"os/user" "os/user"
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings"
"github.com/AlecAivazis/survey/v2" "github.com/AlecAivazis/survey/v2"
"github.com/crowdsecurity/crowdsec/pkg/metabase" "github.com/crowdsecurity/crowdsec/pkg/metabase"
@ -225,8 +226,9 @@ cscli dashboard remove --force
} }
if force { if force {
if err := metabase.RemoveImageContainer(); err != nil { if err := metabase.RemoveImageContainer(); err != nil {
log.Fatalf("removing docker image: %s", err) if !strings.Contains(err.Error(), "No such image") {
log.Fatalf("removing docker image: %s", err)
}
} }
} }
} }

4
cmd/crowdsec-cli/lapi.go
View file

@ -84,6 +84,8 @@ Keep in mind the machine needs to be validated by an administrator on LAPI side
log.Fatalf("api client register: %s", err) log.Fatalf("api client register: %s", err)
} }
log.Printf("Successfully registered to Local API (LAPI)")
var dumpFile string var dumpFile string
if outputFile != "" { if outputFile != "" {
dumpFile = outputFile dumpFile = outputFile
@ -106,7 +108,7 @@ Keep in mind the machine needs to be validated by an administrator on LAPI side
if err != nil { if err != nil {
log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err) log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
} }
log.Printf("API credentials dumped to '%s'", dumpFile) log.Printf("Local API credentials dumped to '%s'", dumpFile)
} else { } else {
fmt.Printf("%s\n", string(apiConfigDump)) fmt.Printf("%s\n", string(apiConfigDump))
} }

2
cmd/crowdsec-cli/machines.go
View file

@ -213,7 +213,7 @@ cscli machines add MyTestMachine --password MyPassword
if err != nil { if err != nil {
log.Fatalf("unable to create machine: %s", err) log.Fatalf("unable to create machine: %s", err)
} }
log.Infof("Machine '%s' created successfully", machineID) log.Infof("Machine '%s' successfully added to the local API", machineID)
if apiURL == "" { if apiURL == "" {
if csConfig.API.Client != nil && csConfig.API.Client.Credentials != nil && csConfig.API.Client.Credentials.URL != "" { if csConfig.API.Client != nil && csConfig.API.Client.Credentials != nil && csConfig.API.Client.Credentials.URL != "" {

5
cmd/crowdsec-cli/main.go
View file

@ -40,7 +40,8 @@ func initConfig() {
} else if err_lvl { } else if err_lvl {
log.SetLevel(log.ErrorLevel) log.SetLevel(log.ErrorLevel)
} }
logFormatter := &log.TextFormatter{TimestampFormat: "02-01-2006 03:04:05 PM", FullTimestamp: true}
log.SetFormatter(logFormatter)
csConfig = csconfig.NewConfig() csConfig = csconfig.NewConfig()
log.Debugf("Using %s as configuration file", ConfigFilePath) log.Debugf("Using %s as configuration file", ConfigFilePath)
@ -65,8 +66,8 @@ func initConfig() {
} }
if csConfig.Cscli.Output == "json" { if csConfig.Cscli.Output == "json" {
log.SetLevel(log.WarnLevel)
log.SetFormatter(&log.JSONFormatter{}) log.SetFormatter(&log.JSONFormatter{})
log.SetLevel(log.ErrorLevel)
} else if csConfig.Cscli.Output == "raw" { } else if csConfig.Cscli.Output == "raw" {
log.SetLevel(log.ErrorLevel) log.SetLevel(log.ErrorLevel)
} }

2
docs/v1.X/docs/observability/prometheus.md
View file

@ -74,7 +74,7 @@ cs_node_hits_ko_total{name="crowdsecurity/http-logs",source="/var/log/nginx/erro
- `cs_lapi_route_requests_total` : number of calls to each route per method - `cs_lapi_route_requests_total` : number of calls to each route per method
- `cs_lapi_machine_requests_total` : number of calls to each route per method grouped by machines - `cs_lapi_machine_requests_total` : number of calls to each route per method grouped by machines
- `cs_lapi_bouncer_requests_total` : number of of calls to each route per method grouped by bouncers - `cs_lapi_bouncer_requests_total` : number of calls to each route per method grouped by bouncers
- `cs_lapi_decisions_ko_total` : number of unsuccessfully responses when bouncers ask for an IP. - `cs_lapi_decisions_ko_total` : number of unsuccessfully responses when bouncers ask for an IP.
- `cs_lapi_decisions_ok_total` : number of successfully responses when bouncers ask for an IP. - `cs_lapi_decisions_ok_total` : number of successfully responses when bouncers ask for an IP.

2
pkg/csconfig/config.go
View file

@ -84,7 +84,7 @@ func (c *GlobalConfig) LoadConfiguration() error {
if c.Crowdsec != nil { if c.Crowdsec != nil {
if c.Crowdsec.AcquisitionFilePath != "" { if c.Crowdsec.AcquisitionFilePath != "" {
log.Infof("non-empty acquisition file path %s", c.Crowdsec.AcquisitionFilePath) log.Debugf("non-empty acquisition file path %s", c.Crowdsec.AcquisitionFilePath)
if _, err := os.Stat(c.Crowdsec.AcquisitionFilePath); err != nil { if _, err := os.Stat(c.Crowdsec.AcquisitionFilePath); err != nil {
return errors.Wrapf(err, "while checking acquisition path %s", c.Crowdsec.AcquisitionFilePath) return errors.Wrapf(err, "while checking acquisition path %s", c.Crowdsec.AcquisitionFilePath)
} }

6
pkg/types/utils.go
View file

@ -41,10 +41,8 @@ func SetDefaultLoggerConfig(cfgMode string, cfgFolder string, cfgLevel log.Level
} }
logLevel = cfgLevel logLevel = cfgLevel
log.SetLevel(logLevel) log.SetLevel(logLevel)
if logLevel >= log.InfoLevel { logFormatter = &log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}
logFormatter = &log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true} log.SetFormatter(logFormatter)
log.SetFormatter(logFormatter)
}
return nil return nil
} }

115
wizard.sh
View file

@ -3,6 +3,7 @@
set -o pipefail set -o pipefail
#set -x #set -x
RED='\033[0;31m' RED='\033[0;31m'
BLUE='\033[0;34m' BLUE='\033[0;34m'
GREEN='\033[0;32m' GREEN='\033[0;32m'
@ -67,19 +68,26 @@ rm -rf $BACKUP_DIR
log_info() { log_info() {
msg=$1 msg=$1
date=$(date +%x:%X) date=$(date +%x:%X)
echo -e "[${date}][${BLUE}INF${NC}] crowdsec_wizard: ${msg}" echo -e "${BLUE}INFO${NC}[${date}] crowdsec_wizard: ${msg}"
}
log_fatal() {
msg=$1
date=$(date +%x:%X)
echo -e "${RED}FATA${NC}[${date}] crowdsec_wizard: ${msg}" 1>&2
exit 1
} }
log_warn() { log_warn() {
msg=$1 msg=$1
date=$(date +%x:%X) date=$(date +%x:%X)
echo -e "[${date}][${ORANGE}WARN${NC}] crowdsec_wizard: ${msg}" echo -e "${ORANGE}WARN${NC}[${date}] crowdsec_wizard: ${msg}"
} }
log_err() { log_err() {
msg=$1 msg=$1
date=$(date +%x:%X) date=$(date +%x:%X)
echo -e "[${date}][${RED}ERR${NC}] crowdsec_wizard: ${msg}" 1>&2 echo -e "${RED}ERR${NC}[${date}] crowdsec_wizard: ${msg}" 1>&2
} }
log_dbg() { log_dbg() {
@ -98,7 +106,7 @@ detect_services () {
#raw ps #raw ps
PSAX=`ps ax -o comm=` PSAX=`ps ax -o comm=`
for SVC in ${SUPPORTED_SERVICES} ; do for SVC in ${SUPPORTED_SERVICES} ; do
log_info "Checking if service '${SVC}' is running (ps+systemd)" log_dbg "Checking if service '${SVC}' is running (ps+systemd)"
for SRC in "${SYSTEMD_SERVICES}" "${PSAX}" ; do for SRC in "${SYSTEMD_SERVICES}" "${PSAX}" ; do
echo ${SRC} | grep ${SVC} >/dev/null echo ${SRC} | grep ${SVC} >/dev/null
if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
@ -108,7 +116,7 @@ detect_services () {
fi fi
DETECTED_SERVICES+=(${SVC}) DETECTED_SERVICES+=(${SVC})
HMENU+=(${SVC} "on") HMENU+=(${SVC} "on")
log_info "Found '${SVC}' running" log_dbg "Found '${SVC}' running"
break; break;
fi; fi;
done; done;
@ -127,9 +135,9 @@ detect_services () {
log_err "user bailed out at services selection" log_err "user bailed out at services selection"
exit 1; exit 1;
fi; fi;
echo "Detected services (interactive) : ${DETECTED_SERVICES[@]}" log_dbg "Detected services (interactive) : ${DETECTED_SERVICES[@]}"
else else
echo "Detected services (unattended) : ${DETECTED_SERVICES[@]}" log_dbg "Detected services (unattended) : ${DETECTED_SERVICES[@]}"
fi; fi;
} }
@ -176,7 +184,7 @@ find_logs_for() {
candidates=`find "${path}" -type f -mtime -5 -ctime -5 -name "$fname"` candidates=`find "${path}" -type f -mtime -5 -ctime -5 -name "$fname"`
#We have some candidates, add them #We have some candidates, add them
for final_file in ${candidates} ; do for final_file in ${candidates} ; do
log_info "Found logs file for '${SVC}': ${final_file}" log_dbg "Found logs file for '${SVC}': ${final_file}"
DETECTED_LOGFILES+=(${final_file}) DETECTED_LOGFILES+=(${final_file})
HMENU+=(${final_file} "on") HMENU+=(${final_file} "on")
done; done;
@ -264,15 +272,16 @@ genyaml() {
echo "labels:" >> ${TMP_ACQUIS_FILE} echo "labels:" >> ${TMP_ACQUIS_FILE}
echo " "${log_input_tags[${service}]} >> ${TMP_ACQUIS_FILE} echo " "${log_input_tags[${service}]} >> ${TMP_ACQUIS_FILE}
echo "---" >> ${TMP_ACQUIS_FILE} echo "---" >> ${TMP_ACQUIS_FILE}
log_info "Acquisition file generated" log_dbg "tmp acquisition file generated to: ${TMP_ACQUIS_FILE}"
} }
genacquisition() { genacquisition() {
log_info "Found following services : "${DETECTED_SERVICES[@]} log_dbg "Found following services : "${DETECTED_SERVICES[@]}
for PSVG in ${DETECTED_SERVICES[@]} ; do for PSVG in ${DETECTED_SERVICES[@]} ; do
find_logs_for ${PSVG} find_logs_for ${PSVG}
if [[ ${#DETECTED_LOGFILES[@]} -gt 0 ]] ; then if [[ ${#DETECTED_LOGFILES[@]} -gt 0 ]] ; then
genyaml ${PSVG} ${DETECTED_LOGFILES[@]} log_info "service '${PSVG}': ${DETECTED_LOGFILES[*]}"
genyaml ${PSVG} ${DETECTED_LOGFILES[@]}
fi; fi;
done done
} }
@ -357,21 +366,22 @@ install_crowdsec() {
#tmp #tmp
mkdir -p /tmp/data mkdir -p /tmp/data
mkdir -p /etc/crowdsec/hub/ mkdir -p /etc/crowdsec/hub/
install -v -m 600 -D "./config/${CLIENT_SECRETS}" "${CROWDSEC_CONFIG_PATH}" || exit install -v -m 600 -D "./config/${CLIENT_SECRETS}" "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit
install -v -m 600 -D "./config/${LAPI_SECRETS}" "${CROWDSEC_CONFIG_PATH}" || exit install -v -m 600 -D "./config/${LAPI_SECRETS}" "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit
## end tmp ## end tmp
install -v -m 644 -D ./config/config.yaml "${CROWDSEC_CONFIG_PATH}" || exit install -v -m 644 -D ./config/config.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit
install -v -m 644 -D ./config/dev.yaml "${CROWDSEC_CONFIG_PATH}" || exit install -v -m 644 -D ./config/dev.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit
install -v -m 644 -D ./config/user.yaml "${CROWDSEC_CONFIG_PATH}" || exit install -v -m 644 -D ./config/user.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit
install -v -m 644 -D ./config/acquis.yaml "${CROWDSEC_CONFIG_PATH}" || exit install -v -m 644 -D ./config/acquis.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit
install -v -m 644 -D ./config/profiles.yaml "${CROWDSEC_CONFIG_PATH}" || exit install -v -m 644 -D ./config/profiles.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit
install -v -m 644 -D ./config/simulation.yaml "${CROWDSEC_CONFIG_PATH}" || exit install -v -m 644 -D ./config/simulation.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit
mkdir -p ${PID_DIR} || exit mkdir -p ${PID_DIR} || exit
PID=${PID_DIR} DATA=${CROWDSEC_DATA_DIR} CFG=${CROWDSEC_CONFIG_PATH} envsubst '$CFG $PID $DATA' < ./config/user.yaml > ${CROWDSEC_CONFIG_PATH}"/user.yaml" PID=${PID_DIR} DATA=${CROWDSEC_DATA_DIR} CFG=${CROWDSEC_CONFIG_PATH} envsubst '$CFG $PID $DATA' < ./config/user.yaml > ${CROWDSEC_CONFIG_PATH}"/user.yaml" || log_fatal "unable to generate user configuration file"
if [[ ${DOCKER_MODE} == "false" ]]; then if [[ ${DOCKER_MODE} == "false" ]]; then
CFG=${CROWDSEC_CONFIG_PATH} PID=${PID_DIR} BIN=${CROWDSEC_BIN_INSTALLED} envsubst '$CFG $PID $BIN' < ./config/crowdsec.service > "${SYSTEMD_PATH_FILE}" CFG=${CROWDSEC_CONFIG_PATH} PID=${PID_DIR} BIN=${CROWDSEC_BIN_INSTALLED} envsubst '$CFG $PID $BIN' < ./config/crowdsec.service > "${SYSTEMD_PATH_FILE}" || log_fatal "unable to crowdsec systemd file"
fi fi
install_bins install_bins
@ -385,7 +395,7 @@ update_bins() {
delete_bins delete_bins
install_bins install_bins
log_info "Upgrade finished" log_info "Upgrade finished"
systemctl restart crowdsec systemctl restart crowdsec || log_fatal "unable to restart crowdsec with systemctl"
} }
update_full() { update_full() {
@ -415,13 +425,13 @@ update_full() {
cp ${BACKUP_DIR}/crowdsec.db /var/lib/crowdsec/data/crowdsec.db cp ${BACKUP_DIR}/crowdsec.db /var/lib/crowdsec/data/crowdsec.db
fi fi
log_info "Finished, restarting" log_info "Finished, restarting"
systemctl restart crowdsec || log_err "Failed to restart crowdsec" systemctl restart crowdsec || log_fatal "Failed to restart crowdsec"
} }
install_bins() { install_bins() {
log_info "Installing crowdsec binaries" log_dbg "Installing crowdsec binaries"
install -v -m 755 -D "${CROWDSEC_BIN}" "${CROWDSEC_BIN_INSTALLED}" || exit install -v -m 755 -D "${CROWDSEC_BIN}" "${CROWDSEC_BIN_INSTALLED}" 1> /dev/null || exit
install -v -m 755 -D "${CSCLI_BIN}" "${CSCLI_BIN_INSTALLED}" || exit install -v -m 755 -D "${CSCLI_BIN}" "${CSCLI_BIN_INSTALLED}" 1> /dev/null || exit
symlink_bins symlink_bins
} }
@ -456,10 +466,9 @@ check_running_bouncers() {
# uninstall crowdsec and cscli # uninstall crowdsec and cscli
uninstall_crowdsec() { uninstall_crowdsec() {
systemctl stop crowdsec.service 1>/dev/null
systemctl stop crowdsec.service systemctl disable -q crowdsec.service 1>/dev/null
systemctl disable crowdsec.service ${CSCLI_BIN} dashboard remove -f -y >/dev/null
${CSCLI_BIN} dashboard remove -f -y
delete_bins delete_bins
# tmp # tmp
@ -480,13 +489,33 @@ function show_link {
echo "" echo ""
echo "Useful links to start with Crowdsec:" echo "Useful links to start with Crowdsec:"
echo "" echo ""
echo " - Documentation : https://docs.crowdsec.net/" echo " - Documentation : https://docs.crowdsec.net/Crowdsec/v1/getting_started/crowdsec-tour/"
echo " - Crowdsec Hub : https://hub.crowdsec.net/ " echo " - Crowdsec Hub : https://hub.crowdsec.net/ "
echo " - Open issues : https://github.com/crowdsecurity/crowdsec/issues" echo " - Open issues : https://github.com/crowdsecurity/crowdsec/issues"
echo "" echo ""
echo "Useful commands to start with Crowdsec:"
echo ""
echo " - sudo cscli metrics : https://docs.crowdsec.net/Crowdsec/v1/cscli/cscli_metrics/"
echo " - sudo cscli decisions list : https://docs.crowdsec.net/Crowdsec/v1/cscli/cscli_decisions_list/"
echo " - sudo cscli alerts list : https://docs.crowdsec.net/Crowdsec/v1/cscli/cscli_alerts_list/"
echo " - sudo cscli hub list : https://docs.crowdsec.net/Crowdsec/v1/cscli/cscli_hub_list/"
echo ""
} }
main() { main() {
if [ "$1" == "install" ] || [ "$1" == "configure" ]; then
if [ "${SILENT}" == "false" ]; then
which whiptail > /dev/null
if [ $? -ne 0 ]; then
log_fatal "whiptail binary is needed to use the wizard in interactive mode, exiting ..."
fi
fi
which envsubst > /dev/null
if [ $? -ne 0 ]; then
log_fatal "envsubst binary is needed to use do a full install with the wizard, exiting ..."
fi
fi
if [[ "$1" == "binupgrade" ]]; if [[ "$1" == "binupgrade" ]];
then then
if ! [ $(id -u) = 0 ]; then if ! [ $(id -u) = 0 ]; then
@ -561,12 +590,12 @@ main() {
log_err "Please run the wizard as root or with sudo" log_err "Please run the wizard as root or with sudo"
exit 1 exit 1
fi fi
log_info "checking existing crowdsec install" log_info "checking if crowdsec is installed"
detect_cs_install detect_cs_install
## Do make build before installing (as non--root) in order to have the binary and then install crowdsec as root ## Do make build before installing (as non--root) in order to have the binary and then install crowdsec as root
log_info "installing crowdsec" log_info "installing crowdsec"
install_crowdsec install_crowdsec
log_info "configuring ${CSCLI_BIN_INSTALLED}" log_dbg "configuring ${CSCLI_BIN_INSTALLED}"
${CSCLI_BIN_INSTALLED} hub update > /dev/null 2>&1 || (log_err "fail to update crowdsec hub. exiting" && exit 1) ${CSCLI_BIN_INSTALLED} hub update > /dev/null 2>&1 || (log_err "fail to update crowdsec hub. exiting" && exit 1)
# detect running services # detect running services
@ -579,27 +608,27 @@ main() {
# Generate acquisition file and move it to the right folder # Generate acquisition file and move it to the right folder
genacquisition genacquisition
mv "${TMP_ACQUIS_FILE}" "${ACQUIS_TARGET}" mv "${TMP_ACQUIS_FILE}" "${ACQUIS_TARGET}"
log_info "acquisition file path: ${ACQUIS_TARGET}"
# Install collections according to detected services # Install collections according to detected services
log_info "Installing needed collections ..." log_dbg "Installing needed collections ..."
install_collection install_collection
# install patterns/ folder # install patterns/ folder
log_info "Installing patterns" log_dbg "Installing patterns"
mkdir -p "${PATTERNS_PATH}" mkdir -p "${PATTERNS_PATH}"
cp "./${PATTERNS_FOLDER}/"* "${PATTERNS_PATH}/" cp "./${PATTERNS_FOLDER}/"* "${PATTERNS_PATH}/"
# api register # api register
${CSCLI_BIN_INSTALLED} machines add --force "$(cat /etc/machine-id)" --password "$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)" -f "${CROWDSEC_CONFIG_PATH}/${CLIENT_SECRETS}" ${CSCLI_BIN_INSTALLED} machines add --force "$(cat /etc/machine-id)" --password "$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)" -f "${CROWDSEC_CONFIG_PATH}/${CLIENT_SECRETS}" || log_fatal "unable to add machine to the local API"
log_info "Crowdsec LAPI registered" log_dbg "Crowdsec LAPI registered"
${CSCLI_BIN_INSTALLED} capi register ${CSCLI_BIN_INSTALLED} capi register || log_fatal "unable to register to the Central API"
log_info "Crowdsec CAPI registered" log_dbg "Crowdsec CAPI registered"
systemctl enable crowdsec systemctl enable -q crowdsec >/dev/null || log_fatal "unable to enable crowdsec"
systemctl start crowdsec systemctl start crowdsec >/dev/null || log_fatal "unable to start crowdsec"
log_info "Enabling and starting crowdsec daemon" log_info "enabling and starting crowdsec daemon"
show_link show_link
return return
fi fi