From 8b504e9f6705063d67fc6bbd08f3605253b93bcd Mon Sep 17 00:00:00 2001 From: AlteredCoder <64792091+AlteredCoder@users.noreply.github.com> Date: Thu, 25 Feb 2021 11:20:36 +0100 Subject: [PATCH] improve logging in cscli and wizard (#643) --- cmd/crowdsec-cli/capi.go | 2 +- cmd/crowdsec-cli/dashboard.go | 6 +- cmd/crowdsec-cli/lapi.go | 4 +- cmd/crowdsec-cli/machines.go | 2 +- cmd/crowdsec-cli/main.go | 5 +- docs/v1.X/docs/observability/prometheus.md | 2 +- pkg/csconfig/config.go | 2 +- pkg/types/utils.go | 6 +- wizard.sh | 115 +++++++++++++-------- 9 files changed, 88 insertions(+), 56 deletions(-) diff --git a/cmd/crowdsec-cli/capi.go b/cmd/crowdsec-cli/capi.go index d80a5e9d9..05e95b6be 100644 --- a/cmd/crowdsec-cli/capi.go +++ b/cmd/crowdsec-cli/capi.go @@ -91,7 +91,7 @@ func NewCapiCmd() *cobra.Command { if err != nil { log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err) } - log.Printf("API credentials dumped to '%s'", dumpFile) + log.Printf("Central API credentials dumped to '%s'", dumpFile) } else { fmt.Printf("%s\n", string(apiConfigDump)) } diff --git a/cmd/crowdsec-cli/dashboard.go b/cmd/crowdsec-cli/dashboard.go index c503e3e88..da25b2794 100644 --- a/cmd/crowdsec-cli/dashboard.go +++ b/cmd/crowdsec-cli/dashboard.go @@ -7,6 +7,7 @@ import ( "os/user" "path/filepath" "strconv" + "strings" "github.com/AlecAivazis/survey/v2" "github.com/crowdsecurity/crowdsec/pkg/metabase" @@ -225,8 +226,9 @@ cscli dashboard remove --force } if force { if err := metabase.RemoveImageContainer(); err != nil { - log.Fatalf("removing docker image: %s", err) - + if !strings.Contains(err.Error(), "No such image") { + log.Fatalf("removing docker image: %s", err) + } } } } diff --git a/cmd/crowdsec-cli/lapi.go b/cmd/crowdsec-cli/lapi.go index e6140e6eb..04d6ff697 100644 --- a/cmd/crowdsec-cli/lapi.go +++ b/cmd/crowdsec-cli/lapi.go @@ -84,6 +84,8 @@ Keep in mind the machine needs to be validated by an administrator on LAPI side log.Fatalf("api client register: %s", err) } + log.Printf("Successfully registered to Local API (LAPI)") + var dumpFile string if outputFile != "" { dumpFile = outputFile @@ -106,7 +108,7 @@ Keep in mind the machine needs to be validated by an administrator on LAPI side if err != nil { log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err) } - log.Printf("API credentials dumped to '%s'", dumpFile) + log.Printf("Local API credentials dumped to '%s'", dumpFile) } else { fmt.Printf("%s\n", string(apiConfigDump)) } diff --git a/cmd/crowdsec-cli/machines.go b/cmd/crowdsec-cli/machines.go index 9c49db59a..bf23952b1 100644 --- a/cmd/crowdsec-cli/machines.go +++ b/cmd/crowdsec-cli/machines.go @@ -213,7 +213,7 @@ cscli machines add MyTestMachine --password MyPassword if err != nil { log.Fatalf("unable to create machine: %s", err) } - log.Infof("Machine '%s' created successfully", machineID) + log.Infof("Machine '%s' successfully added to the local API", machineID) if apiURL == "" { if csConfig.API.Client != nil && csConfig.API.Client.Credentials != nil && csConfig.API.Client.Credentials.URL != "" { diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index f24aec3f0..a9d0e3d51 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -40,7 +40,8 @@ func initConfig() { } else if err_lvl { log.SetLevel(log.ErrorLevel) } - + logFormatter := &log.TextFormatter{TimestampFormat: "02-01-2006 03:04:05 PM", FullTimestamp: true} + log.SetFormatter(logFormatter) csConfig = csconfig.NewConfig() log.Debugf("Using %s as configuration file", ConfigFilePath) @@ -65,8 +66,8 @@ func initConfig() { } if csConfig.Cscli.Output == "json" { - log.SetLevel(log.WarnLevel) log.SetFormatter(&log.JSONFormatter{}) + log.SetLevel(log.ErrorLevel) } else if csConfig.Cscli.Output == "raw" { log.SetLevel(log.ErrorLevel) } diff --git a/docs/v1.X/docs/observability/prometheus.md b/docs/v1.X/docs/observability/prometheus.md index 18729aa7b..3ca013309 100644 --- a/docs/v1.X/docs/observability/prometheus.md +++ b/docs/v1.X/docs/observability/prometheus.md @@ -74,7 +74,7 @@ cs_node_hits_ko_total{name="crowdsecurity/http-logs",source="/var/log/nginx/erro - `cs_lapi_route_requests_total` : number of calls to each route per method - `cs_lapi_machine_requests_total` : number of calls to each route per method grouped by machines - - `cs_lapi_bouncer_requests_total` : number of of calls to each route per method grouped by bouncers + - `cs_lapi_bouncer_requests_total` : number of calls to each route per method grouped by bouncers - `cs_lapi_decisions_ko_total` : number of unsuccessfully responses when bouncers ask for an IP. - `cs_lapi_decisions_ok_total` : number of successfully responses when bouncers ask for an IP. diff --git a/pkg/csconfig/config.go b/pkg/csconfig/config.go index 769687fa5..7dd6144a7 100644 --- a/pkg/csconfig/config.go +++ b/pkg/csconfig/config.go @@ -84,7 +84,7 @@ func (c *GlobalConfig) LoadConfiguration() error { if c.Crowdsec != nil { if c.Crowdsec.AcquisitionFilePath != "" { - log.Infof("non-empty acquisition file path %s", c.Crowdsec.AcquisitionFilePath) + log.Debugf("non-empty acquisition file path %s", c.Crowdsec.AcquisitionFilePath) if _, err := os.Stat(c.Crowdsec.AcquisitionFilePath); err != nil { return errors.Wrapf(err, "while checking acquisition path %s", c.Crowdsec.AcquisitionFilePath) } diff --git a/pkg/types/utils.go b/pkg/types/utils.go index c394ffff8..6ffdf71c6 100644 --- a/pkg/types/utils.go +++ b/pkg/types/utils.go @@ -41,10 +41,8 @@ func SetDefaultLoggerConfig(cfgMode string, cfgFolder string, cfgLevel log.Level } logLevel = cfgLevel log.SetLevel(logLevel) - if logLevel >= log.InfoLevel { - logFormatter = &log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true} - log.SetFormatter(logFormatter) - } + logFormatter = &log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true} + log.SetFormatter(logFormatter) return nil } diff --git a/wizard.sh b/wizard.sh index 6a2674170..210190fb0 100755 --- a/wizard.sh +++ b/wizard.sh @@ -3,6 +3,7 @@ set -o pipefail #set -x + RED='\033[0;31m' BLUE='\033[0;34m' GREEN='\033[0;32m' @@ -67,19 +68,26 @@ rm -rf $BACKUP_DIR log_info() { msg=$1 date=$(date +%x:%X) - echo -e "[${date}][${BLUE}INF${NC}] crowdsec_wizard: ${msg}" + echo -e "${BLUE}INFO${NC}[${date}] crowdsec_wizard: ${msg}" +} + +log_fatal() { + msg=$1 + date=$(date +%x:%X) + echo -e "${RED}FATA${NC}[${date}] crowdsec_wizard: ${msg}" 1>&2 + exit 1 } log_warn() { msg=$1 date=$(date +%x:%X) - echo -e "[${date}][${ORANGE}WARN${NC}] crowdsec_wizard: ${msg}" + echo -e "${ORANGE}WARN${NC}[${date}] crowdsec_wizard: ${msg}" } log_err() { msg=$1 date=$(date +%x:%X) - echo -e "[${date}][${RED}ERR${NC}] crowdsec_wizard: ${msg}" 1>&2 + echo -e "${RED}ERR${NC}[${date}] crowdsec_wizard: ${msg}" 1>&2 } log_dbg() { @@ -98,7 +106,7 @@ detect_services () { #raw ps PSAX=`ps ax -o comm=` for SVC in ${SUPPORTED_SERVICES} ; do - log_info "Checking if service '${SVC}' is running (ps+systemd)" + log_dbg "Checking if service '${SVC}' is running (ps+systemd)" for SRC in "${SYSTEMD_SERVICES}" "${PSAX}" ; do echo ${SRC} | grep ${SVC} >/dev/null if [ $? -eq 0 ]; then @@ -108,7 +116,7 @@ detect_services () { fi DETECTED_SERVICES+=(${SVC}) HMENU+=(${SVC} "on") - log_info "Found '${SVC}' running" + log_dbg "Found '${SVC}' running" break; fi; done; @@ -127,9 +135,9 @@ detect_services () { log_err "user bailed out at services selection" exit 1; fi; - echo "Detected services (interactive) : ${DETECTED_SERVICES[@]}" + log_dbg "Detected services (interactive) : ${DETECTED_SERVICES[@]}" else - echo "Detected services (unattended) : ${DETECTED_SERVICES[@]}" + log_dbg "Detected services (unattended) : ${DETECTED_SERVICES[@]}" fi; } @@ -176,7 +184,7 @@ find_logs_for() { candidates=`find "${path}" -type f -mtime -5 -ctime -5 -name "$fname"` #We have some candidates, add them for final_file in ${candidates} ; do - log_info "Found logs file for '${SVC}': ${final_file}" + log_dbg "Found logs file for '${SVC}': ${final_file}" DETECTED_LOGFILES+=(${final_file}) HMENU+=(${final_file} "on") done; @@ -264,15 +272,16 @@ genyaml() { echo "labels:" >> ${TMP_ACQUIS_FILE} echo " "${log_input_tags[${service}]} >> ${TMP_ACQUIS_FILE} echo "---" >> ${TMP_ACQUIS_FILE} - log_info "Acquisition file generated" + log_dbg "tmp acquisition file generated to: ${TMP_ACQUIS_FILE}" } genacquisition() { - log_info "Found following services : "${DETECTED_SERVICES[@]} + log_dbg "Found following services : "${DETECTED_SERVICES[@]} for PSVG in ${DETECTED_SERVICES[@]} ; do find_logs_for ${PSVG} if [[ ${#DETECTED_LOGFILES[@]} -gt 0 ]] ; then - genyaml ${PSVG} ${DETECTED_LOGFILES[@]} + log_info "service '${PSVG}': ${DETECTED_LOGFILES[*]}" + genyaml ${PSVG} ${DETECTED_LOGFILES[@]} fi; done } @@ -357,21 +366,22 @@ install_crowdsec() { #tmp mkdir -p /tmp/data mkdir -p /etc/crowdsec/hub/ - install -v -m 600 -D "./config/${CLIENT_SECRETS}" "${CROWDSEC_CONFIG_PATH}" || exit - install -v -m 600 -D "./config/${LAPI_SECRETS}" "${CROWDSEC_CONFIG_PATH}" || exit + install -v -m 600 -D "./config/${CLIENT_SECRETS}" "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit + install -v -m 600 -D "./config/${LAPI_SECRETS}" "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit ## end tmp - install -v -m 644 -D ./config/config.yaml "${CROWDSEC_CONFIG_PATH}" || exit - install -v -m 644 -D ./config/dev.yaml "${CROWDSEC_CONFIG_PATH}" || exit - install -v -m 644 -D ./config/user.yaml "${CROWDSEC_CONFIG_PATH}" || exit - install -v -m 644 -D ./config/acquis.yaml "${CROWDSEC_CONFIG_PATH}" || exit - install -v -m 644 -D ./config/profiles.yaml "${CROWDSEC_CONFIG_PATH}" || exit - install -v -m 644 -D ./config/simulation.yaml "${CROWDSEC_CONFIG_PATH}" || exit + install -v -m 644 -D ./config/config.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit + install -v -m 644 -D ./config/dev.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit + install -v -m 644 -D ./config/user.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit + install -v -m 644 -D ./config/acquis.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit + install -v -m 644 -D ./config/profiles.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit + install -v -m 644 -D ./config/simulation.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit + mkdir -p ${PID_DIR} || exit - PID=${PID_DIR} DATA=${CROWDSEC_DATA_DIR} CFG=${CROWDSEC_CONFIG_PATH} envsubst '$CFG $PID $DATA' < ./config/user.yaml > ${CROWDSEC_CONFIG_PATH}"/user.yaml" + PID=${PID_DIR} DATA=${CROWDSEC_DATA_DIR} CFG=${CROWDSEC_CONFIG_PATH} envsubst '$CFG $PID $DATA' < ./config/user.yaml > ${CROWDSEC_CONFIG_PATH}"/user.yaml" || log_fatal "unable to generate user configuration file" if [[ ${DOCKER_MODE} == "false" ]]; then - CFG=${CROWDSEC_CONFIG_PATH} PID=${PID_DIR} BIN=${CROWDSEC_BIN_INSTALLED} envsubst '$CFG $PID $BIN' < ./config/crowdsec.service > "${SYSTEMD_PATH_FILE}" + CFG=${CROWDSEC_CONFIG_PATH} PID=${PID_DIR} BIN=${CROWDSEC_BIN_INSTALLED} envsubst '$CFG $PID $BIN' < ./config/crowdsec.service > "${SYSTEMD_PATH_FILE}" || log_fatal "unable to crowdsec systemd file" fi install_bins @@ -385,7 +395,7 @@ update_bins() { delete_bins install_bins log_info "Upgrade finished" - systemctl restart crowdsec + systemctl restart crowdsec || log_fatal "unable to restart crowdsec with systemctl" } update_full() { @@ -415,13 +425,13 @@ update_full() { cp ${BACKUP_DIR}/crowdsec.db /var/lib/crowdsec/data/crowdsec.db fi log_info "Finished, restarting" - systemctl restart crowdsec || log_err "Failed to restart crowdsec" + systemctl restart crowdsec || log_fatal "Failed to restart crowdsec" } install_bins() { - log_info "Installing crowdsec binaries" - install -v -m 755 -D "${CROWDSEC_BIN}" "${CROWDSEC_BIN_INSTALLED}" || exit - install -v -m 755 -D "${CSCLI_BIN}" "${CSCLI_BIN_INSTALLED}" || exit + log_dbg "Installing crowdsec binaries" + install -v -m 755 -D "${CROWDSEC_BIN}" "${CROWDSEC_BIN_INSTALLED}" 1> /dev/null || exit + install -v -m 755 -D "${CSCLI_BIN}" "${CSCLI_BIN_INSTALLED}" 1> /dev/null || exit symlink_bins } @@ -456,10 +466,9 @@ check_running_bouncers() { # uninstall crowdsec and cscli uninstall_crowdsec() { - - systemctl stop crowdsec.service - systemctl disable crowdsec.service - ${CSCLI_BIN} dashboard remove -f -y + systemctl stop crowdsec.service 1>/dev/null + systemctl disable -q crowdsec.service 1>/dev/null + ${CSCLI_BIN} dashboard remove -f -y >/dev/null delete_bins # tmp @@ -480,13 +489,33 @@ function show_link { echo "" echo "Useful links to start with Crowdsec:" echo "" - echo " - Documentation : https://docs.crowdsec.net/" + echo " - Documentation : https://docs.crowdsec.net/Crowdsec/v1/getting_started/crowdsec-tour/" echo " - Crowdsec Hub : https://hub.crowdsec.net/ " echo " - Open issues : https://github.com/crowdsecurity/crowdsec/issues" echo "" + echo "Useful commands to start with Crowdsec:" + echo "" + echo " - sudo cscli metrics : https://docs.crowdsec.net/Crowdsec/v1/cscli/cscli_metrics/" + echo " - sudo cscli decisions list : https://docs.crowdsec.net/Crowdsec/v1/cscli/cscli_decisions_list/" + echo " - sudo cscli alerts list : https://docs.crowdsec.net/Crowdsec/v1/cscli/cscli_alerts_list/" + echo " - sudo cscli hub list : https://docs.crowdsec.net/Crowdsec/v1/cscli/cscli_hub_list/" + echo "" } main() { + if [ "$1" == "install" ] || [ "$1" == "configure" ]; then + if [ "${SILENT}" == "false" ]; then + which whiptail > /dev/null + if [ $? -ne 0 ]; then + log_fatal "whiptail binary is needed to use the wizard in interactive mode, exiting ..." + fi + fi + which envsubst > /dev/null + if [ $? -ne 0 ]; then + log_fatal "envsubst binary is needed to use do a full install with the wizard, exiting ..." + fi + fi + if [[ "$1" == "binupgrade" ]]; then if ! [ $(id -u) = 0 ]; then @@ -561,12 +590,12 @@ main() { log_err "Please run the wizard as root or with sudo" exit 1 fi - log_info "checking existing crowdsec install" + log_info "checking if crowdsec is installed" detect_cs_install ## Do make build before installing (as non--root) in order to have the binary and then install crowdsec as root log_info "installing crowdsec" install_crowdsec - log_info "configuring ${CSCLI_BIN_INSTALLED}" + log_dbg "configuring ${CSCLI_BIN_INSTALLED}" ${CSCLI_BIN_INSTALLED} hub update > /dev/null 2>&1 || (log_err "fail to update crowdsec hub. exiting" && exit 1) # detect running services @@ -579,27 +608,27 @@ main() { # Generate acquisition file and move it to the right folder genacquisition mv "${TMP_ACQUIS_FILE}" "${ACQUIS_TARGET}" - + log_info "acquisition file path: ${ACQUIS_TARGET}" # Install collections according to detected services - log_info "Installing needed collections ..." + log_dbg "Installing needed collections ..." install_collection # install patterns/ folder - log_info "Installing patterns" + log_dbg "Installing patterns" mkdir -p "${PATTERNS_PATH}" cp "./${PATTERNS_FOLDER}/"* "${PATTERNS_PATH}/" # api register - ${CSCLI_BIN_INSTALLED} machines add --force "$(cat /etc/machine-id)" --password "$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)" -f "${CROWDSEC_CONFIG_PATH}/${CLIENT_SECRETS}" - log_info "Crowdsec LAPI registered" + ${CSCLI_BIN_INSTALLED} machines add --force "$(cat /etc/machine-id)" --password "$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)" -f "${CROWDSEC_CONFIG_PATH}/${CLIENT_SECRETS}" || log_fatal "unable to add machine to the local API" + log_dbg "Crowdsec LAPI registered" - ${CSCLI_BIN_INSTALLED} capi register - log_info "Crowdsec CAPI registered" + ${CSCLI_BIN_INSTALLED} capi register || log_fatal "unable to register to the Central API" + log_dbg "Crowdsec CAPI registered" - systemctl enable crowdsec - systemctl start crowdsec - log_info "Enabling and starting crowdsec daemon" + systemctl enable -q crowdsec >/dev/null || log_fatal "unable to enable crowdsec" + systemctl start crowdsec >/dev/null || log_fatal "unable to start crowdsec" + log_info "enabling and starting crowdsec daemon" show_link return fi