Func tests (#970)

* stabilize func tests
* fix cold logs func tests
* add wait condition on mock server

Co-authored-by: sabban <15465465+sabban@users.noreply.github.com>

scripts/func_tests/tests_base.sh

@@ -29,6 +29,17 @@ function pathadd {
     fi
 }
 
+function wait_for_service {
+    count=0
+    while ! nc -z localhost 6060; do   
+        sleep 0.5
+        ((count ++))
+        if [[ count == 21 ]]; then
+            fail "$@"
+        fi
+    done
+}
+
 pathadd /usr/sbin
 
 if [ -f /etc/systemd/system/crowdsec.service ]; then

scripts/func_tests/tests_post-install_0base.sh

@@ -24,14 +24,12 @@ pidof crowdsec && fail "crowdsec process shouldn't be running"
 #start it again
 ${SYSTEMCTL} start crowdsec || fail "failed to stop service"
 ${SYSTEMCTL} status crowdsec || fail "crowdsec should be up"
-sleep 5s
-pidof crowdsec || fail "crowdsec process should be running"
+wait_for_service "crowdsec process should be running"
 
 #restart it
 ${SYSTEMCTL} restart crowdsec || fail "failed to stop service"
 ${SYSTEMCTL} status crowdsec || fail "crowdsec should be up"
-sleep 5s
-pidof crowdsec || fail "crowdsec process should be running"
+wait_for_service "crowdsec process should be running"
 
 ## version
 ${CSCLI} version || fail "cannot run cscli version"
@@ -106,7 +104,7 @@ sudo mv /tmp/crowdsec.service /etc/systemd/system/crowdsec.service
 
 ${SYSTEMCTL} daemon-reload
 ${SYSTEMCTL} start crowdsec 
-pidof crowdsec || fail "crowdsec LAPI should run without agent (in flag)"
+wait_for_service "crowdsec LAPI should run without agent (in flag)"
 ${SYSTEMCTL} stop crowdsec
 
 sed '/^ExecStart/s/-no-cs//g' ${SYSTEMD_SERVICE_FILE} > /tmp/crowdsec.service
@@ -117,7 +115,7 @@ ${SYSTEMCTL} daemon-reload
 # test with no crowdsec agent in configuration file
 sudo cp ./config/config_no_agent.yaml /etc/crowdsec/config.yaml
 ${SYSTEMCTL} start crowdsec 
-pidof crowdsec || fail "crowdsec LAPI should run without agent (in configuration file)"
+wait_for_service "crowdsec LAPI should run without agent (in configuration file)"
 
 
 ## capi
@@ -142,7 +140,7 @@ echo "CROWDSEC (AGENT+LAPI)"
 # test with no online client in configuration file
 sudo cp ./config/config_no_capi.yaml /etc/crowdsec/config.yaml
 ${SYSTEMCTL} start crowdsec 
-pidof crowdsec || fail "crowdsec LAPI should run without CAPI (in configuration file)"
+wait_for_service "crowdsec LAPI should run without CAPI (in configuration file)"
 
 ## capi
 ${CSCLI} -c ./config/config_no_capi.yaml capi status && fail "capi status should not be ok" ## if capi status success, it means that the test fail
@@ -161,3 +159,4 @@ sudo cp ./config.yaml.backup /etc/crowdsec/config.yaml
 
 ${SYSTEMCTL} daemon-reload
 ${SYSTEMCTL} restart crowdsec
+wait_for_service "crowdsec should be restarted)"

scripts/func_tests/tests_post-install_4cold-logs.sh

@@ -14,10 +14,14 @@ ${SYSTEMCTL} reload crowdsec
 # generate a fake bf log -> cold logs processing
 rm  -f ssh-bf.log
 
+sync
+
 for i in `seq 1 10` ; do 
     echo `LC_ALL=C date '+%b %d %H:%M:%S '`'sd-126005 sshd[12422]: Invalid user netflix from 1.1.1.172 port 35424' >> ssh-bf.log
 done;
 
+sync
+
 ${CROWDSEC} -dsn "file://./ssh-bf.log" -type syslog -no-api
 
 ${CSCLI} decisions list -o=json | ${JQ} '. | length == 1' || fail "expected exactly one decision"
@@ -31,6 +35,7 @@ ${CSCLI} decisions list -i 1.1.1.173 -o=json | ${JQ} '. == null' || fail "(exact
 
 ${CSCLI} decisions delete --all
 
+sudo cp /etc/crowdsec/acquis.yaml ./acquis.yaml.backup
 echo "" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
 echo "filename: /tmp/test.log" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
 echo "labels:" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
@@ -38,10 +43,19 @@ echo "  type: syslog" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
 touch /tmp/test.log
 
 ${SYSTEMCTL} restart crowdsec
-sleep 1
+wait_for_service "crowdsec should run (cold logs)"
 ${SYSTEMCTL} status crowdsec
+
+sleep 2s
+
 cat ssh-bf.log >> /tmp/test.log
 
-sleep 2
-
+sleep 5s
 ${CSCLI} decisions list -o=json | ${JQ} '.[].decisions[0].value == "1.1.1.172"' || fail "(live) expected ban on 1.1.1.172"
+
+sudo cp ./acquis.yaml.backup /etc/crowdsec/acquis.yaml
+
+sync
+
+${SYSTEMCTL} restart crowdsec
+wait_for_service "crowdsec should run"

scripts/func_tests/tests_post-install_5simulation.sh

@@ -16,17 +16,25 @@ ${SYSTEMCTL} reload crowdsec
 # generate a fake bf log -> cold logs processing
 rm  -f ssh-bf.log
 
+sync
+
 for i in `seq 1 10` ; do 
     echo `LC_ALL=C date '+%b %d %H:%M:%S '`'sd-126005 sshd[12422]: Invalid user netflix from 1.1.1.174 port 35424' >> ssh-bf.log
 done;
 
+sync
+
 ${CROWDSEC} -dsn file://./ssh-bf.log -type syslog -no-api
 
+sleep 1s
+
 ${CSCLI} decisions list -o=json | ${JQ} '. | length == 1' || fail "expected exactly one decision"
 ${CSCLI} decisions list -o=json | ${JQ} '.[].decisions[0].value == "1.1.1.174"'  || fail "(exact) expected ban on 1.1.1.174"
 ${CSCLI} decisions list -o=json | ${JQ} '.[].decisions[0].simulated == false'  || fail "(exact) expected simulated on false"
 
 
+sleep 1s
+
 # enable simulation on specific scenario and try with same logs
 
 ${CSCLI} decisions delete --all
@@ -36,6 +44,7 @@ ${CROWDSEC} -dsn file://./ssh-bf.log -type syslog -no-api
 
 ${CSCLI} decisions list --no-simu -o=json | ${JQ} '. == null' || fail "expected no decision (listing only non-simulated decisions)"
 
+sleep 1s
 # enable global simulation and try with same logs
 
 ${CSCLI} decisions delete --all
@@ -44,4 +53,5 @@ ${CSCLI} simulation enable --global
 
 ${CROWDSEC} -dsn file://./ssh-bf.log -type syslog -no-api
 
+sleep 1s
 ${CSCLI} decisions list --no-simu -o=json | ${JQ} '. == null' || fail "expected no decision (listing only non-simulated decisions)"

scripts/func_tests/tests_post-install_7_plugin.sh

@@ -36,6 +36,15 @@ function setup_tests() {
     cscli decisions delete --all
     modify_config
     python3 -u mock_http_server.py > mock_http_server_logs.log &
+    count=0
+    while ! nc -z localhost 9999; do   
+        sleep 0.5
+        ((count ++))
+        if [[ count == 41 ]]; then
+            fail "mock server not up after 20s"
+        fi
+    done
+
     MOCK_SERVER_PID=$!
 }
 
@@ -55,7 +64,7 @@ function run_tests() {
         cleanup_tests
         fail "expected 0 log lines fom mock http server before adding decisions"
     fi
-    sleep 2s
+    sleep 5s
     ${CSCLI} decisions add --ip 1.2.3.4 --duration 30s
     ${CSCLI} decisions add --ip 1.2.3.5 --duration 30s
     sleep 5s