tests/bin cleanup (#1760)

.github/workflows/bats-hub.yml

@@ -52,7 +52,7 @@ jobs:
       run: make bats-test-hub
 
     - name: "Collect hub coverage"
-      run: ./tests/collect-hub-coverage >> $GITHUB_ENV
+      run: ./tests/bin/collect-hub-coverage >> $GITHUB_ENV
 
     - name: "Create Parsers badge"
       uses: schneegans/dynamic-badges-action@v1.1.0

tests/README.md

@@ -396,6 +396,11 @@ A mysql-client package is required as well.
 
 ## troubleshooting
 
+ - CAPI is disabled, why?
+Most tests don't need it. Helper scripts are provided in `tests/enable-capi`
+and `tests/disable-capi` for interactive use, and two library functions
+`config_enable_capi` and `config_disable_capi` to call inside the tests.
+You still need to call `cscli capi register` after enabling it.
 
  - My tests are hanging forever, why?
 See if you have a jq/yq or similar process waiting for standard input. Hint:

tests/bats.mk

@@ -28,8 +28,8 @@ PLUGIN_DIR = $(LOCAL_DIR)/lib/crowdsec/plugins
 DB_BACKEND ?= sqlite
 
 ifdef TEST_COVERAGE
-  CROWDSEC = $(TEST_DIR)/crowdsec-wrapper
-  CSCLI = $(TEST_DIR)/cscli-wrapper
+  CROWDSEC = $(TEST_DIR)/bin/crowdsec-wrapper
+  CSCLI = $(TEST_DIR)/bin/cscli-wrapper
   BINCOVER_TESTING = true
 else
   # the wrappers should work here too - it detects TEST_COVERAGE - but we allow
@@ -71,7 +71,7 @@ bats-environment:
 
 # Verify dependencies and submodules
 bats-check-requirements:
-	@$(TEST_DIR)/check-requirements
+	@$(TEST_DIR)/bin/check-requirements
 
 # Build and installs crowdsec in a local directory. Rebuilds if already exists.
 bats-build: bats-environment bats-check-requirements
@@ -100,7 +100,7 @@ bats-test: bats-environment bats-check-requirements
 
 # Generate dynamic tests
 bats-test-hub: bats-environment bats-check-requirements
-	@$(TEST_DIR)/generate-hub-tests
+	@$(TEST_DIR)/bin/generate-hub-tests
 	$(TEST_DIR)/run-tests $(TEST_DIR)/dyn-bats
 
 # Static checks for the test scripts.

tests/bats/05_config_yaml_local.bats

@@ -56,28 +56,28 @@ teardown() {
     # disable the agent or we'll need to patch api client credentials too
     run -0 config_disable_agent
     ./instance-crowdsec start
-    run -0 ./lib/util/wait-for-port -q 8080
+    run -0 ./bin/wait-for-port -q 8080
     ./instance-crowdsec stop
-    run -1 ./lib/util/wait-for-port -q 8080
+    run -1 ./bin/wait-for-port -q 8080
 
     echo "{'api':{'server':{'listen_uri':127.0.0.1:8083}}}" >"${CONFIG_YAML}.local"
 
     ./instance-crowdsec start
-    run -0 ./lib/util/wait-for-port -q 8083
-    run -1 ./lib/util/wait-for-port -q 8080
+    run -0 ./bin/wait-for-port -q 8083
+    run -1 ./bin/wait-for-port -q 8080
     ./instance-crowdsec stop
 
     rm -f "${CONFIG_YAML}.local"
     ./instance-crowdsec start
-    run -1 ./lib/util/wait-for-port -q 8083
-    run -0 ./lib/util/wait-for-port -q 8080
+    run -1 ./bin/wait-for-port -q 8083
+    run -0 ./bin/wait-for-port -q 8080
 }
 
 @test "local_api_credentials.yaml.local" {
     run -0 config_disable_agent
     echo "{'api':{'server':{'listen_uri':127.0.0.1:8083}}}" >"${CONFIG_YAML}.local"
     ./instance-crowdsec start
-    run -0 ./lib/util/wait-for-port -q 8083
+    run -0 ./bin/wait-for-port -q 8083
 
     run -1 cscli decisions list
     echo "{'url':'http://127.0.0.1:8083'}" >"${LOCAL_API_CREDENTIALS}.local"

tests/bin/check-requirements

@@ -8,14 +8,13 @@ die() {
 }
 
 # shellcheck disable=SC1007
-TEST_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
-# shellcheck source=./.environment.sh
-. "${TEST_DIR}/.environment.sh"
-
+THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+# shellcheck source=../.environment.sh
+. "${THIS_DIR}/../.environment.sh"
 
 check_bats_core() {
     if ! "${TEST_DIR}/lib/bats-core/bin/bats" --version >/dev/null 2>&1; then
-        die "ERROR: bats-core submodule is required. Please run 'git submodule init; git submodule update' and retry."
+        die "ERROR: the bats-* submodules are required. Please run 'git submodule init; git submodule update' and retry."
     fi
 }
 

tests/bin/collect-hub-coverage

@@ -8,9 +8,9 @@ die() {
 }
 
 # shellcheck disable=SC1007
-TEST_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
-# shellcheck source=./.environment.sh
-. "${TEST_DIR}/.environment.sh"
+THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+# shellcheck disable=SC1091
+. "${THIS_DIR}/../.environment.sh"
 
 hubdir="${LOCAL_DIR}/hub-tests"
 

tests/bin/crowdsec-wrapper

@@ -9,8 +9,8 @@ set -eu
 #shellcheck disable=SC1007
 THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
 # no need to change directory, and doing it here would break hub tests
-#shellcheck disable=SC1090
-. "${THIS_DIR}/.environment.sh"
+# shellcheck disable=SC1091
+. "${THIS_DIR}/../.environment.sh"
 
 set -o pipefail  # don't let sed hide the statuscode
 mkdir -p "${LOCAL_DIR}/var/lib/coverage"

tests/bin/cscli-wrapper

@@ -9,8 +9,8 @@ set -eu
 #shellcheck disable=SC1007
 THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
 # no need to change directory, and doing it here would break hub tests
-#shellcheck disable=SC1090
-. "${THIS_DIR}/.environment.sh"
+# shellcheck disable=SC1091
+. "${THIS_DIR}/../.environment.sh"
 
 set -o pipefail  # don't let sed hide the statuscode
 mkdir -p "${LOCAL_DIR}/var/lib/coverage"

tests/bin/generate-hub-tests

@@ -3,9 +3,9 @@
 set -eu
 
 # shellcheck disable=SC1007
-TEST_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
-# shellcheck source=./.environment.sh
-. "${TEST_DIR}/.environment.sh"
+THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+# shellcheck disable=SC1091
+. "${THIS_DIR}/../.environment.sh"
 
 cscli() {
     "${CSCLI}" "$@"
@@ -22,11 +22,11 @@ cat << EOT > "${HUBTESTS_BATS}"
 set -u
 
 setup_file() {
-    load "../lib/setup_file.sh" >&3 2>&1
+    load "../lib/setup_file.sh"
 }
 
 teardown_file() {
-    load "../lib/teardown_file.sh" >&3 2>&1
+    load "../lib/teardown_file.sh"
 }
 
 setup() {

tests/disable-capi

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# shellcheck disable=SC1007
+THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+# shellcheck disable=SC1091
+. "${THIS_DIR}/.environment.sh"
+
+yq e 'del(.api.server.online_client)' -i "${CONFIG_YAML}"

tests/enable-capi

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# shellcheck disable=SC1007
+THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+# shellcheck disable=SC1091
+. "${THIS_DIR}/.environment.sh"
+
+online_api_credentials="$(dirname "${CONFIG_YAML}")/online_api_credentials.yaml"
+export online_api_credentials
+
+yq e '.api.server.online_client.credentials_path=strenv(online_api_credentials)' -i "${CONFIG_YAML}"

tests/instance-crowdsec

@@ -3,7 +3,7 @@
 #shellcheck disable=SC1007
 THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
 cd "${THIS_DIR}" || exit 1
-#shellcheck disable=SC1090
+# shellcheck disable=SC1091
 . ./.environment.sh
 
 backend_script="./lib/init/crowdsec-${INIT_BACKEND}"

tests/instance-data

@@ -3,7 +3,7 @@
 #shellcheck disable=SC1007
 THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
 cd "${THIS_DIR}" || exit 1
-#shellcheck disable=SC1090
+# shellcheck disable=SC1091
 . ./.environment.sh
 
 backend_script="./lib/config/config-${CONFIG_BACKEND}"

tests/instance-db

@@ -3,10 +3,10 @@
 #shellcheck disable=SC1007
 THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
 cd "${THIS_DIR}" || exit 1
-#shellcheck disable=SC1090
+# shellcheck disable=SC1091
 . ./.environment.sh
 
-./assert-crowdsec-not-running
+./bin/assert-crowdsec-not-running
 
 backend_script="./lib/db/instance-${DB_BACKEND}"
 

tests/instance-mock-http

@@ -14,14 +14,13 @@ about() {
 #shellcheck disable=SC1007
 THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
 cd "${THIS_DIR}"
-
-#shellcheck disable=SC1090
-. "${THIS_DIR}/.environment.sh"
+# shellcheck disable=SC1091
+. ./.environment.sh
 
 # you have not removed set -u above, have you?
 
-[ -z "${LOG_DIR-}" ] && die "\$LOG_DIR must be defined."
-[ -z "${PID_DIR-}" ] && die "\$PID_DIR must be defined."
+[[ -z "${LOG_DIR-}" ]] && die "\$LOG_DIR must be defined."
+[[ -z "${PID_DIR-}" ]] && die "\$PID_DIR must be defined."
 
 if ! command -v python3 >/dev/null 2>&2; then
     die "The python3 executable is is missing. Please install it and try again."
@@ -30,18 +29,18 @@ fi
 DAEMON_PID=${PID_DIR}/mock-http.pid
 
 start_instance() {
-    [ $# -lt 1 ] && about
+    [[ $# -lt 1 ]] && about
     daemonize \
         -p "${DAEMON_PID}" \
         -e "${LOG_DIR}/mock-http.err" \
         -o "${LOG_DIR}/mock-http.out" \
-        /usr/bin/env python3 -u "${THIS_DIR}/mock-http.py" "$1"
-    ./lib/util/wait-for-port "$1"
+        /usr/bin/env python3 -u "${THIS_DIR}/bin/mock-http.py" "$1"
+    ./bin/wait-for-port "$1"
 #    echo "mock http started on port $1"
 }
 
 stop_instance() {
-    if [ -f "${DAEMON_PID}" ]; then
+    if [[ -f "${DAEMON_PID}" ]]; then
         # terminate with extreme prejudice, all the application data will be thrown away anyway
         kill -9 "$(cat "${DAEMON_PID}")" > /dev/null 2>&1
         rm -f -- "${DAEMON_PID}"
@@ -51,7 +50,7 @@ stop_instance() {
 
 # ---------------------------
 
-[ $# -lt 1 ] && about
+[[ $# -lt 1 ]] && about
 
 case "$1" in
     start)

tests/lib/config/config-global

@@ -46,14 +46,14 @@ else
 fi
 
 remove_init_data() {
-    ./assert-crowdsec-not-running
+    ./bin/assert-crowdsec-not-running
     rm -rf -- "${LOCAL_DIR:?}/${REL_CONFIG_DIR}"/* "${LOCAL_DIR:?}/${REL_DATA_DIR:?}"/*
 }
 
 # we need a separate function for initializing config when testing package
 # because we want to test the configuration as well
 make_init_data() {
-    ./assert-crowdsec-not-running
+    ./bin/assert-crowdsec-not-running
 
     ./instance-db config-yaml
     ./instance-db setup
@@ -78,7 +78,7 @@ make_init_data() {
 }
 
 load_init_data() {
-    ./assert-crowdsec-not-running
+    ./bin/assert-crowdsec-not-running
 
     if [[ ! -f "${LOCAL_INIT_DIR}/init-config-data.tar" ]]; then
         die "Initial data not found; did you run '${script_name} make' ?"

tests/lib/config/config-local

@@ -46,7 +46,7 @@ else
 fi
 
 remove_init_data() {
-    ./assert-crowdsec-not-running
+    ./bin/assert-crowdsec-not-running
     rm -rf -- "${LOCAL_DIR:?}/${REL_CONFIG_DIR}"/* "${LOCAL_DIR:?}/${REL_DATA_DIR:?}"/*
 }
 
@@ -91,7 +91,7 @@ config_generate() {
 
 
 make_init_data() {
-    ./assert-crowdsec-not-running
+    ./bin/assert-crowdsec-not-running
 
     remove_init_data
     mkdir -p "${DATA_DIR}"
@@ -125,7 +125,7 @@ make_init_data() {
 }
 
 load_init_data() {
-    ./assert-crowdsec-not-running
+    ./bin/assert-crowdsec-not-running
 
     if [[ ! -f "${LOCAL_INIT_DIR}/init-config-data.tar" ]]; then
         die "Initial data not found; did you run '${script_name} make' ?"

tests/lib/init/crowdsec-daemon

@@ -36,7 +36,7 @@ start() {
         -e "${LOG_DIR}/crowdsec.err" \
         -o "${LOG_DIR}/crowdsec.out" \
         "${CROWDSEC}"
-    ./lib/util/wait-for-port 6060
+    ./bin/wait-for-port 6060
 }
 
 stop() {

tests/lib/init/crowdsec-systemd

@@ -32,7 +32,7 @@ fi
 
 start() {
     systemctl start crowdsec
-    ./lib/util/wait-for-port 6060
+    ./bin/wait-for-port 6060
 }
 
 stop() {

tests/lib/setup_file.sh

@@ -22,7 +22,7 @@ eval "$(debug)"
 cd "${TEST_DIR}"
 
 # complain if there's a crowdsec running system-wide or leftover from a previous test
-./assert-crowdsec-not-running
+./bin/assert-crowdsec-not-running
 
 # we can prepend the filename to the test descriptions (useful to feed a TAP consumer)
 if [[ "${PREFIX_TEST_NAMES_WITH_FILE:-false}" == "true" ]]; then

tests/run-tests

@@ -12,7 +12,7 @@ TEST_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
 # shellcheck source=./.environment.sh
 . "${TEST_DIR}/.environment.sh"
 
-"${TEST_DIR}/check-requirements"
+"${TEST_DIR}/bin/check-requirements"
 
 echo "Running tests..."
 echo "DB_BACKEND: ${DB_BACKEND}"
@@ -24,7 +24,6 @@ else
     mkdir -p "${LOCAL_DIR}/var/lib/coverage"
 fi
 
-
 dump_backend="$(cat "${LOCAL_INIT_DIR}/.backend")"
 if [[ "${DB_BACKEND}" != "${dump_backend}" ]]; then
     die "Can't run with backend '${DB_BACKEND}' because the test data was build with '${dump_backend}'"