Unified functional tests (#696)

* remove dead / not needed code & scripts

* don't rely on /dev/urandom in wizard

* unified functional tests

.github/workflows/ci_functests-install.yml

@@ -1,13 +1,11 @@
-name: Functionnal tests install
+name: Functional tests
 
 on:
   push:
     branches:
-      - wip_lapi
       - master
   pull_request:
     branches:
-      - wip_lapi
       - master
 
 jobs:
@@ -33,64 +31,34 @@ jobs:
     - name: Install release
       run: |
         cd crowdsec-${{ steps.keydb.outputs.release }}
-        sudo bash -x ./wizard.sh --bininstall
-        sudo cscli machines add -a
-    - name: Post-installation check
+        sudo ./wizard.sh --unattended
+    - name: "Test post-install base"
       run: |
-        sudo cscli hub update
-    - name: Install collection
+          cd scripts/func_tests/
+          ./tests_post-install_0base.sh
+    - name: "Test post-install bouncer"
       run: |
-        sudo cscli hub list -a
-        sudo cscli parsers install crowdsecurity/syslog-logs crowdsecurity/sshd-logs crowdsecurity/dateparse-enrich
-        sudo cscli scenarios install crowdsecurity/ssh-bf
-    - name: Crowdsec start service
+          cd scripts/func_tests/
+          ./tests_post-install_1bouncers.sh
+    - name: "Test post-install bouncer"
       run: |
-        sudo systemctl start crowdsec
-    - name: Generate fake ssh bf logs
+          cd scripts/func_tests/
+          ./tests_post-install_2collections.sh
+    - name: "Test post-install bouncer"
       run: |
-        for i in `seq 1 10` ; do 
-          echo `date '+%b %d %H:%M:%S '`'sd-126005 sshd[12422]: Invalid user netflix from 1.1.1.172 port 35424' >> ssh-bf.log
-        done;
-    - name: Process ssh-bf logs in time-machine
+          cd scripts/func_tests/
+          ./tests_post-install_3machines.sh   
+    - name: "Test post-install ip management"
       run: |
-        sudo crowdsec -file ./ssh-bf.log -type syslog -no-api
-    - name: Cscli ban list check
-      #check that we got the expected ban and that the filters are working properly
+          cd scripts/func_tests/
+          ./tests_post-install_99ip_mgmt.sh
+    - name: "Test cold logs"
       run: |
-        sudo cscli decisions list
-        sudo cscli decisions list -o=json | jq -e '.[].decisions[0].value == "1.1.1.172"'
-        sudo cscli decisions list -r 1.1.1.0/24 -o=json --contained | jq -e '.[].decisions[0].value == "1.1.1.172"'
-        sudo cscli decisions list -r 1.1.2.0/24 -o=json | jq -e '. == null'
-        sudo cscli decisions list -i 1.1.1.172 -o=json | jq -e '.[].decisions[0].value == "1.1.1.172"'
-        sudo cscli decisions list -i 1.1.1.173 -o=json | jq -e '. == null'
-    - name: Cscli ban del check
-      #check that the delete is working and that filters are working properly
+          cd scripts/func_tests/
+          ./tests_post-install_4cold-logs.sh
+    - name: "Uninstall"
+      run: sudo ./wizard.sh --uninstall
+    - name: "Test post remove"
       run: |
-        sudo cscli decisions delete -i 1.1.1.173
-        sudo cscli decisions list -o=json | jq -e '.[].decisions[0].value == "1.1.1.172"'
-        sudo cscli decisions delete -i 1.1.1.172
-        sudo cscli decisions list -o=json | jq -e '. == null'
-    - name: Metrics check
-      run: |
-        sudo cscli metrics
-    - name: Service stop & config change
-      #shutdown the service, edit that acquisition.yaml
-      run: |
-        sudo systemctl stop crowdsec
-        echo "" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
-        echo "filename: /tmp/test.log" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
-        echo "labels:" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
-        echo "  type: syslog" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
-        touch /tmp/test.log
-    - name: Service start & check
-      run: |
-        sudo systemctl start crowdsec || sudo journalctl -xe
-    - name: Trigger events via normal acquisition
-      run: |
-        cat ssh-bf.log >> /tmp/test.log
-        sleep 1
-    - name: Check results
-      run: |
-        sudo cscli decisions list -o=json | jq -e '.[].decisions[0].value == "1.1.1.172"'
-
-
+          cd scripts/func_tests/
+          bash -x ./tests_post-remove_0base.sh      

.github/workflows/ci_functests-ip_mgmt.yml

@@ -1,31 +0,0 @@
-name: Functionnal tests for IP management
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
-
-jobs:
-  build:
-    name: Install generated release and perform basic tests
-    runs-on: ubuntu-latest
-    steps:
-    - name: Set up Go 1.13
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.13
-      id: go
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
-    - name: install ipset for bouncer
-      run: |
-        sudo apt update
-        sudo apt install jq
-    - name: run tests
-      run: |
-        cd scripts/
-        sudo ./test_ip_management.sh
-

.github/workflows/ci_functests-upgrade.yml

@@ -1,39 +0,0 @@
-name: Functionnal tests upgrade
-
-on:
-  push:
-    branches:
-      - add_upgrade_tests
-      - master
-  pull_request:
-    branches:
-      - add_upgrade_tests
-      - master
-
-jobs:
-  build:
-    name: Install generated release and perform basic tests
-    runs-on: ubuntu-latest
-    steps:
-    - name: Set up Go 1.13
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.13
-      id: go
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
-    - name: install ipset for bouncer
-      run: |
-        sudo apt update
-        sudo apt install ipset git
-    - id: keydb
-      uses: pozetroninc/github-action-get-latest-release@master
-      with:
-        owner: crowdsecurity
-        repo: crowdsec
-        excludes: draft
-    - name: run tests
-      run: |
-        cd scripts
-        sudo ./test_wizard_upgrade.sh --version ${{ steps.keydb.outputs.release }}
-

Makefile

@@ -50,17 +50,6 @@ goversion:
         exit 1; \
     fi
 
-hubci:
-	@rm -rf crowdsec-xxx hub-tests
-	BUILD_VERSION=xxx make release
-	@git clone https://github.com/crowdsecurity/hub-tests.git
-	@cd hub-tests && make
-	@cd crowdsec-xxx && ./test_env.sh
-	@cd crowdsec-xxx/tests && bash ../../scripts/install_all.sh
-	@cp hub-tests/main ./crowdsec-xxx/tests/
-	@cp -R hub-tests/tests ./crowdsec-xxx/tests/
-	@cd ./crowdsec-xxx/tests/ && bash ../../hub-tests/run_tests.sh
-
 clean:
 	@$(MAKE) -C $(CROWDSEC_FOLDER) clean --no-print-directory
 	@$(MAKE) -C $(CSCLI_FOLDER) clean --no-print-directory
@@ -113,12 +102,6 @@ else
 	@exit 1;
 endif
 
-.PHONY: uninstall
-uninstall:
-	@rm -rf "$(CFG_PREFIX)" || exit
-	@rm -rf "$(DATA_PREFIX)" || exit
-	@rm -rf "$(SYSTEMD_PATH_FILE)" || exit
-
 .PHONY: check_release
 check_release:
 	@if [ -d $(RELDIR) ]; then echo "$(RELDIR) already exists, abort" ;  exit 1 ; fi

scripts/build_plugins.sh

@@ -1,29 +0,0 @@
-#!/bin/bash
-
-PLUGIN_DIR="./plugins/"
-
-
-goto() {
-    echo "[*] Going to $1"
-    cd $1
-}
-
-
-goto $PLUGIN_DIR
-CURRENT_DIR=$(pwd)
-for path in $(ls);
-do
-    goto $path
-    modules=$(find . -name "*.go")
-    CURRENT_PLUGDIN_DIR=$(pwd)
-    for mod in $modules;
-    do
-        folder=$(dirname $mod) 
-        plugin_file=$(basename -- "$mod")
-        plugin_name=(${plugin_file%.*})
-        echo "[*] Building plugin $plugin_name from $mod"
-        go build -buildmode=plugin -o "$plugin_name.so" $plugin_file
-        goto $CURRENT_PLUGDIN_DIR
-    done
-    goto $CURRENT_DIR
-done

scripts/func_tests/tests_base.sh

@@ -0,0 +1,22 @@
+#! /usr/bin/env bash
+# -*- coding: utf-8 -*-
+
+
+# sourced by other functionnal tests
+
+PACKAGE_PATH="${PACKAGE_PATH:-./crowdsec.deb}"
+
+CSCLI_BIN="cscli"
+CSCLI="sudo ${CSCLI_BIN}"
+JQ="jq -e"
+
+SYSTEMCTL="sudo systemctl --no-pager"
+
+CROWDSEC="sudo crowdsec"
+
+# helpers
+function fail {
+    echo "ACTION FAILED, STOP : $@"
+    caller
+    exit 1
+}

scripts/func_tests/tests_post-install_0base.sh

@@ -0,0 +1,60 @@
+#! /usr/bin/env bash
+# -*- coding: utf-8 -*-
+
+source tests_base.sh
+
+
+
+## status / start / stop
+# service should be up
+pidof crowdsec || fail "crowdsec process shouldn't be running"
+${SYSTEMCTL} status crowdsec || fail "systemctl status crowdsec failed"
+
+#shut it down
+${SYSTEMCTL} stop crowdsec || fail "failed to stop service"
+${SYSTEMCTL} status crowdsec && fail "crowdsec should be down"
+pidof crowdsec && fail "crowdsec process shouldn't be running"
+
+#start it again
+${SYSTEMCTL} start crowdsec || fail "failed to stop service"
+${SYSTEMCTL} status crowdsec || fail "crowdsec should be down"
+pidof crowdsec || fail "crowdsec process shouldn't be running"
+
+#restart it
+${SYSTEMCTL} restart crowdsec || fail "failed to stop service"
+${SYSTEMCTL} status crowdsec || fail "crowdsec should be down"
+pidof crowdsec || fail "crowdsec process shouldn't be running"
+
+
+
+
+## version
+
+${CSCLI} version || fail "cannot run cscli version"
+
+
+## alerts
+
+# alerts list at startup should just return one entry : comunity pull
+sleep 5
+${CSCLI} alerts list -ojson  | ${JQ} '. | length >= 1' || fail "expected at least one entry from cscli alerts list"
+
+
+## capi
+
+${CSCLI} capi status || fail "capi status should be ok"
+
+
+## config
+
+${CSCLI} config show || fail "failed to show config"
+
+${CSCLI} config backup ./test || fail "failed to backup config"
+
+## lapi
+
+${CSCLI} lapi status || fail "lapi status failed"
+
+## metrics
+${CSCLI} metrics || fail "failed to get metrics"
+

scripts/func_tests/tests_post-install_1bouncers.sh

@@ -0,0 +1,27 @@
+#! /usr/bin/env bash
+# -*- coding: utf-8 -*-
+
+source tests_base.sh
+
+
+## bouncers
+
+# we should have 0 bouncers
+${CSCLI} bouncers list -ojson  | ${JQ} '. | length == 0' || fail "expected 0 bouncers"
+
+# we can add one bouncer - should we save token for later ?
+${CSCLI} bouncers add ciTestBouncer || fail "failed to add bouncer"
+
+# but we can't add it twice - we would get an error
+${CSCLI} bouncers add ciTestBouncer -ojson  2>&1 | ${JQ} '.level == "error"' || fail "didn't receive the expected error"
+
+# we should have 1 bouncer
+${CSCLI} bouncers list -ojson  | ${JQ} '. | length == 1' || fail "expected 1 bouncers"
+
+# delete the bouncer :)
+${CSCLI} bouncers delete ciTestBouncer || fail "failed to delete bouncer"
+
+# we should have 0 bouncers
+${CSCLI} bouncers list -ojson  | ${JQ} '. | length == 0' || fail "expected 0 bouncers"
+
+

scripts/func_tests/tests_post-install_2collections.sh

@@ -0,0 +1,30 @@
+#! /usr/bin/env bash
+# -*- coding: utf-8 -*-
+
+source tests_base.sh
+
+## collections
+
+${CSCLI} collections list || fail "failed to list collections"
+
+BASE_COLLECTION_COUNT=2
+
+# we expect 1 collections : linux 
+${CSCLI} collections list -ojson | ${JQ} ". | length == ${BASE_COLLECTION_COUNT}" || fail "(first) expected exactly ${BASE_COLLECTION_COUNT} collection"
+
+# install an extra collection
+${CSCLI} collections install crowdsecurity/mysql || fail "failed to install collection"
+
+BASE_COLLECTION_COUNT=$(($BASE_COLLECTION_COUNT+1))
+
+# we should now have 2 collections :)
+${CSCLI} collections list -ojson | ${JQ} ". | length == ${BASE_COLLECTION_COUNT}" || fail "(post install) expected exactly ${BASE_COLLECTION_COUNT} collection"
+
+# remove the collection
+${CSCLI} collections remove crowdsecurity/mysql || fail "failed to remove collection"
+
+BASE_COLLECTION_COUNT=$(($BASE_COLLECTION_COUNT-1))
+
+# we expect 1 collections : linux 
+${CSCLI} collections list -ojson | ${JQ} ". | length == ${BASE_COLLECTION_COUNT}" || fail "(post remove) expected exactly ${BASE_COLLECTION_COUNT} collection"
+

scripts/func_tests/tests_post-install_3machines.sh

@@ -0,0 +1,22 @@
+#! /usr/bin/env bash
+# -*- coding: utf-8 -*-
+
+source tests_base.sh
+
+## machines
+
+${CSCLI} machines list -ojson | ${JQ} '. | length == 1' || fail "expected exactly one machine"
+
+# add a new machine
+${CSCLI} machines add -a -f ./test_machine.yaml CiTestMachine -ojson || fail "expected exactly one machine"
+${CSCLI} machines list -ojson | ${JQ} '. | length == 2' || fail "expected exactly one machine"
+${CSCLI} machines delete CiTestMachine -ojson || fail "expected exactly one machine"
+${CSCLI} machines list -ojson | ${JQ} '. | length == 1' || fail "expected exactly one machine"
+
+#try register/validate
+${CSCLI} lapi register  --machine CiTestMachineRegister -f new_machine.yaml
+#the newly added machine isn't validated yet
+${CSCLI} machines list -ojson | ${JQ} '.[1].isValidated == null' || fail "machine shouldn't be validated"
+${CSCLI} machines validate CiTestMachineRegister  || fail "failed to validate machine"
+${CSCLI} machines list -ojson | ${JQ} '.[1].isValidated == true' || fail "machine should be validated"
+

scripts/func_tests/tests_post-install_4cold-logs.sh

@@ -0,0 +1,47 @@
+#! /usr/bin/env bash
+# -*- coding: utf-8 -*-
+
+source tests_base.sh
+
+
+# install sshd collection
+
+${CSCLI} collections install crowdsecurity/sshd
+${CSCLI} decisions delete --all
+${SYSTEMCTL} reload crowdsec
+
+
+# generate a fake bf log -> cold logs processing
+rm  -f ssh-bf.log
+
+for i in `seq 1 10` ; do 
+    echo `date '+%b %d %H:%M:%S '`'sd-126005 sshd[12422]: Invalid user netflix from 1.1.1.172 port 35424' >> ssh-bf.log
+done;
+
+${CROWDSEC} -file ./ssh-bf.log -type syslog -no-api
+
+${CSCLI} decisions list -o=json | ${JQ} '. | length == 1' || fail "expected exactly one decision"
+${CSCLI} decisions list -o=json | ${JQ} '.[].decisions[0].value == "1.1.1.172"'  || fail "(exact) expected ban on 1.1.1.172"
+${CSCLI} decisions list -r 1.1.1.0/24 -o=json --contained | ${JQ} '.[].decisions[0].value == "1.1.1.172"' || fail "(range/contained) expected ban on 1.1.1.172"
+${CSCLI} decisions list -r 1.1.2.0/24 -o=json | ${JQ} '. == null'  || fail "(range/NOT-contained) expected no ban on 1.1.1.172"
+${CSCLI} decisions list -i 1.1.1.172 -o=json | ${JQ} '.[].decisions[0].value == "1.1.1.172"'  || fail "(range/NOT-contained) expected ban on 1.1.1.172"
+${CSCLI} decisions list -i 1.1.1.173 -o=json | ${JQ} '. == null' || fail "(exact) expected no ban on 1.1.1.173"
+
+# generate a live ssh bf
+
+${CSCLI} decisions delete --all
+
+echo "" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
+echo "filename: /tmp/test.log" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
+echo "labels:" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
+echo "  type: syslog" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
+touch /tmp/test.log
+
+${SYSTEMCTL} restart crowdsec
+sleep 1
+${SYSTEMCTL} status crowdsec
+cat ssh-bf.log >> /tmp/test.log
+
+sleep 2
+
+${CSCLI} decisions list -o=json | ${JQ} '.[].decisions[0].value == "1.1.1.172"' || fail "(live) expected ban on 1.1.1.172"

scripts/func_tests/tests_post-install_99ip_mgmt.sh

@@ -2,6 +2,9 @@
 # -*- coding: utf-8 -*-
 
 
+source tests_base.sh
+
+
 # Codes
 RED='\033[0;31m'
 GREEN='\033[0;32m'
@@ -9,9 +12,7 @@ NC='\033[0m'
 OK_STR="${GREEN}OK${NC}"
 FAIL_STR="${RED}FAIL${NC}"
 
-CSCLI_BIN="./cscli"
-CSCLI="${CSCLI_BIN} -c dev.yaml"
-JQ="jq -e"
+
 CROWDSEC_API_URL="http://localhost:8081"
 CROWDSEC_VERSION=""
 API_KEY=""
@@ -20,11 +21,6 @@ RELEASE_FOLDER_FULL=""
 FAILED="false"
 MUST_FAIL="false"
 
-
-get_latest_release() {
-  CROWDSEC_VERSION=$(curl --silent "https://api.github.com/repos/crowdsecurity/crowdsec/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
-}
-
 ### Helpers
 function docurl
 {
@@ -52,39 +48,6 @@ function cscli_echo {
     FAILED="false"
 }
 
-function fail {
-    FAILED="true"
-    MUST_FAIL="true"
-}
-
-## End helpers ##
-
-
-function init
-{
-    if [[ ! -d ${RELEASE_FOLDER} ]];
-    then
-      cd ..
-      get_latest_release
-      BUILD_VERSION=${CROWDSEC_VERSION} make release
-      if [ $? != 0 ]; then
-        echo "Unable to make the release (make sur you have go installed), exiting"
-        exit 1
-      fi
-      RELEASE_FOLDER="crowdsec-${CROWDSEC_VERSION}"
-    fi
-    RELEASE_FOLDER_FULL="$(readlink -f ${RELEASE_FOLDER})"
-    TEST_ENV_FOLDER="${RELEASE_FOLDER_FULL}/tests/"
-    cd ${RELEASE_FOLDER}/
-    if [[ ! -d "${TEST_ENV_FOLDER}" ]];
-    then
-        echo "Installing crowdsec test environment"
-        ./test_env.sh
-    fi
-    reset
-}
-
-
 function test_ipv4_ip
 {
     echo ""
@@ -145,6 +108,7 @@ function test_ipv4_range
     echo "##########################################"
     echo ""
 
+
     cscli_echo "adding decision for range 4.4.4.0/24"
     ${CSCLI} decisions add -r 4.4.4.0/24 > /dev/null 2>&1 || fail
 
@@ -390,36 +354,17 @@ function start_test
 {
 
     ## ipv4 testing
+    ${CSCLI} decisions delete --all
+
     test_ipv4_ip
     test_ipv4_range
 
-    reset
-
     ## ipv6 testing
+    ${CSCLI} decisions delete --all
     test_ipv6_ip
     test_ipv6_range
 }
 
-function reset 
-{
-    cd ${RELEASE_FOLDER_FULL}/tests/
-    killall -w crowdsec > /dev/null 2>&1 || echo ""
-    rm data/crowdsec.db > /dev/null 2>&1 || echo ""
-    ${CSCLI} hub update
-    ${CSCLI} machines add -a
-    API_KEY=`${CSCLI} bouncers add TestingBouncer -o=raw`
-    ./crowdsec -c dev.yaml> crowdsec-out.log 2>&1  &
-    sleep 2
-}
-
-function down
-{
-  cd ${RELEASE_FOLDER_FULL}/tests/
-  rm data/crowdsec.db
-  killall crowdsec
-  #rm -rf tests/
-}
-
 
 usage() {
       echo "Usage:"
@@ -453,9 +398,7 @@ do
 done
 
 
-init
 start_test
-down
 
 if [[ "${MUST_FAIL}" == "true" ]];
 then

scripts/func_tests/tests_post-remove_0base.sh

@@ -0,0 +1,7 @@
+#! /usr/bin/env bash
+# -*- coding: utf-8 -*-
+
+source tests_base.sh
+
+pidof crowdsec && fail "crowdsec shouldn't run anymore" || true
+

scripts/install_all.sh

@@ -1,7 +0,0 @@
-#!/bin/sh
-
-./cscli -c dev.yaml list parser list -a -o json | jq -r ".[].name" > installed_parsers.txt
-cat installed_parsers.txt | while read parser; do
-    echo "install ${parser}" ;
-    ./cscli -c dev.yaml parsers install ${parser} ;
-done

wizard.sh

@@ -623,7 +623,7 @@ main() {
 
 
         # api register
-        ${CSCLI_BIN_INSTALLED} machines add --force "$(cat /etc/machine-id)" --password "$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)" -f "${CROWDSEC_CONFIG_PATH}/${CLIENT_SECRETS}" || log_fatal "unable to add machine to the local API"
+        ${CSCLI_BIN_INSTALLED} machines add --force "$(cat /etc/machine-id)" -a -f "${CROWDSEC_CONFIG_PATH}/${CLIENT_SECRETS}" || log_fatal "unable to add machine to the local API"
         log_dbg "Crowdsec LAPI registered" 
         
         ${CSCLI_BIN_INSTALLED} capi register || log_fatal "unable to register to the Central API"