From 15a9e16530450786c6f5b3c5284030098053b6d6 Mon Sep 17 00:00:00 2001
From: mmetc <92726601+mmetc@users.noreply.github.com>
Date: Fri, 5 Aug 2022 10:54:49 +0200
Subject: [PATCH] functional tests: json, stderr helpers (#1704)

---
 tests/bats/01_base.bats             | 67 +++++++++++------------------
 tests/bats/01_crowdsec.bats         | 28 ++++--------
 tests/bats/02_nolapi.bats           | 23 ++++------
 tests/bats/03_noagent.bats          | 18 +++-----
 tests/bats/04_nocapi.bats           | 27 +++---------
 tests/bats/20_collections.bats      | 21 ++++++++-
 tests/bats/72_plugin_badconfig.bats | 35 +++++----------
 tests/bats/80_alerts.bats           |  7 +--
 tests/bats/90_decisions.bats        |  6 +--
 tests/lib/setup_file.sh             | 50 +++++++++++++++++++++
 10 files changed, 137 insertions(+), 145 deletions(-)

diff --git a/tests/bats/01_base.bats b/tests/bats/01_base.bats
index 40d8ec7eb..0e318ab7b 100644
--- a/tests/bats/01_base.bats
+++ b/tests/bats/01_base.bats
@@ -35,10 +35,9 @@ declare stderr
 
     # no "usage" output after every error
     run -1 --separate-stderr cscli blahblah
-    run -0 echo "${stderr}"
     # error is displayed as log entry, not with print
-    assert_output --partial 'level=fatal msg="unknown command \"blahblah\" for \"cscli\""'
-    refute_output --partial 'unknown command "blahblah" for "cscli"'
+    assert_stderr --partial 'level=fatal msg="unknown command \"blahblah\" for \"cscli\""'
+    refute_stderr --partial 'unknown command "blahblah" for "cscli"'
 }
 
 @test "cscli version" {
@@ -101,8 +100,7 @@ declare stderr
     ONLINE_API_CREDENTIALS_YAML="$(config_get '.api.server.online_client.credentials_path')"
     rm "${ONLINE_API_CREDENTIALS_YAML}"
     run -1 --separate-stderr cscli capi status
-    run -0 echo "${stderr}"
-    assert_output --partial "Local API is disabled, please run this command on the local API machine: loading online client credentials: failed to read api server credentials configuration file '${ONLINE_API_CREDENTIALS_YAML}': open ${ONLINE_API_CREDENTIALS_YAML}: no such file or directory"
+    assert_stderr --partial "Local API is disabled, please run this command on the local API machine: loading online client credentials: failed to read api server credentials configuration file '${ONLINE_API_CREDENTIALS_YAML}': open ${ONLINE_API_CREDENTIALS_YAML}: no such file or directory"
 }
 
 @test "cscli config show -o human" {
@@ -147,12 +145,10 @@ declare stderr
     # test that we need a valid path
     # disabled because in CI, the empty string is not passed as a parameter
     ## run -1 --separate-stderr cscli config backup ""
-    ## run -0 echo "${stderr}"
-    ## assert_output --partial "Failed to backup configurations: directory path can't be empty"
+    ## assert_stderr --partial "Failed to backup configurations: directory path can't be empty"
 
     run -1 --separate-stderr cscli config backup "/dev/null/blah"
-    run -0 echo "${stderr}"
-    assert_output --partial "Failed to backup configurations: while creating /dev/null/blah: mkdir /dev/null/blah: not a directory"
+    assert_stderr --partial "Failed to backup configurations: while creating /dev/null/blah: mkdir /dev/null/blah: not a directory"
 
     # pick a dirpath
     backupdir=$(TMPDIR="${BATS_TEST_TMPDIR}" mktemp -u)
@@ -163,9 +159,8 @@ declare stderr
 
     # don't overwrite an existing backup
     run -1 --separate-stderr cscli config backup "${backupdir}"
-    run -0 echo "${stderr}"
-    assert_output --partial "Failed to backup configurations"
-    assert_output --partial "file exists"
+    assert_stderr --partial "Failed to backup configurations"
+    assert_stderr --partial "file exists"
 
     SIMULATION_YAML="$(config_get '.config_paths.simulation_path')"
 
@@ -180,8 +175,7 @@ declare stderr
     # backup: detect missing files
     rm "${SIMULATION_YAML}"
     run -1 --separate-stderr cscli config backup "${backupdir}"
-    run -0 echo "${stderr}"
-    assert_output --regexp "Failed to backup configurations: failed copy .* to .*: stat .*: no such file or directory"
+    assert_stderr --regexp "Failed to backup configurations: failed copy .* to .*: stat .*: no such file or directory"
     rm -rf -- "${backupdir:?}"
 }
 
@@ -199,47 +193,38 @@ declare stderr
     LOCAL_API_CREDENTIALS=$(config_get '.api.client.credentials_path')
     rm -f "${LOCAL_API_CREDENTIALS}"
     run -1 --separate-stderr cscli lapi status
-    run -0 echo "${stderr}"
-    assert_output --partial "loading api client: while reading yaml file: open ${LOCAL_API_CREDENTIALS}: no such file or directory"
+    assert_stderr --partial "loading api client: while reading yaml file: open ${LOCAL_API_CREDENTIALS}: no such file or directory"
 
     run -1 --separate-stderr cscli alerts list
-    run -0 echo "${stderr}"
-    assert_output --partial "loading api client: while reading yaml file: open ${LOCAL_API_CREDENTIALS}: no such file or directory"
+    assert_stderr --partial "loading api client: while reading yaml file: open ${LOCAL_API_CREDENTIALS}: no such file or directory"
 
     run -1 --separate-stderr cscli decisions list
-    run -0 echo "${stderr}"
-    assert_output --partial "loading api client: while reading yaml file: open ${LOCAL_API_CREDENTIALS}: no such file or directory"
+    assert_stderr --partial "loading api client: while reading yaml file: open ${LOCAL_API_CREDENTIALS}: no such file or directory"
 }
 
 @test "cscli - empty LAPI credentials file" {
     LOCAL_API_CREDENTIALS=$(config_get '.api.client.credentials_path')
     truncate -s 0 "${LOCAL_API_CREDENTIALS}"
     run -1 --separate-stderr cscli lapi status
-    run -0 echo "${stderr}"
-    assert_output --partial "no credentials or URL found in api client configuration '${LOCAL_API_CREDENTIALS}'"
+    assert_stderr --partial "no credentials or URL found in api client configuration '${LOCAL_API_CREDENTIALS}'"
 
     run -1 --separate-stderr cscli alerts list
-    run -0 echo "${stderr}"
-    assert_output --partial "no credentials or URL found in api client configuration '${LOCAL_API_CREDENTIALS}'"
+    assert_stderr --partial "no credentials or URL found in api client configuration '${LOCAL_API_CREDENTIALS}'"
 
     run -1 --separate-stderr cscli decisions list
-    run -0 echo "${stderr}"
-    assert_output --partial "no credentials or URL found in api client configuration '${LOCAL_API_CREDENTIALS}'"
+    assert_stderr --partial "no credentials or URL found in api client configuration '${LOCAL_API_CREDENTIALS}'"
 }
 
 @test "cscli - missing LAPI client settings" {
     config_set 'del(.api.client)'
     run -1 --separate-stderr cscli lapi status
-    run -0 echo "${stderr}"
-    assert_output --partial "loading api client: no API client section in configuration"
+    assert_stderr --partial "loading api client: no API client section in configuration"
 
     run -1 --separate-stderr cscli alerts list
-    run -0 echo "${stderr}"
-    assert_output --partial "loading api client: no API client section in configuration"
+    assert_stderr --partial "loading api client: no API client section in configuration"
 
     run -1 --separate-stderr cscli decisions list
-    run -0 echo "${stderr}"
-    assert_output --partial "loading api client: no API client section in configuration"
+    assert_stderr --partial "loading api client: no API client section in configuration"
 }
 
 @test "cscli - malformed LAPI url" {
@@ -247,19 +232,16 @@ declare stderr
     config_set "${LOCAL_API_CREDENTIALS}" '.url="https://127.0.0.1:-80"'
 
     run -1 --separate-stderr cscli lapi status
-    run -0 echo "${stderr}"
-    assert_output --partial 'parsing api url'
-    assert_output --partial 'invalid port \":-80\" after host'
+    assert_stderr --partial 'parsing api url'
+    assert_stderr --partial 'invalid port \":-80\" after host'
 
     run -1 --separate-stderr cscli alerts list
-    run -0 echo "${stderr}"
-    assert_output --partial 'parsing api url'
-    assert_output --partial 'invalid port \":-80\" after host'
+    assert_stderr --partial 'parsing api url'
+    assert_stderr --partial 'invalid port \":-80\" after host'
 
     run -1 --separate-stderr cscli decisions list
-    run -0 echo "${stderr}"
-    assert_output --partial 'parsing api url'
-    assert_output --partial 'invalid port \":-80\" after host'
+    assert_stderr --partial 'parsing api url'
+    assert_stderr --partial 'invalid port \":-80\" after host'
 }
 
 @test "cscli metrics" {
@@ -268,8 +250,7 @@ declare stderr
     assert_output --partial "ROUTE"
     assert_output --partial '/v1/watchers/login'
 
-    run -0 echo "${stderr}"
-    assert_output --partial "Local Api Metrics:"
+    assert_stderr --partial "Local Api Metrics:"
 }
 
 @test "'cscli completion' with or without configuration file" {
diff --git a/tests/bats/01_crowdsec.bats b/tests/bats/01_crowdsec.bats
index d63ea0c36..4d8f64d15 100644
--- a/tests/bats/01_crowdsec.bats
+++ b/tests/bats/01_crowdsec.bats
@@ -20,39 +20,31 @@ teardown() {
     ./instance-crowdsec stop
 }
 
-# to silence shellcheck
-declare stderr
-
 #----------
 
 @test "crowdsec (usage)" {
     run -0 --separate-stderr timeout 2s "${CROWDSEC}" -h
-    run -0 echo "${stderr}"
-    assert_line --regexp "Usage of .*:"
+    assert_stderr_line --regexp "Usage of .*:"
 
     run -0 --separate-stderr timeout 2s "${CROWDSEC}" --help
-    run -0 echo "${stderr}"
-    assert_line --regexp "Usage of .*:"
+    assert_stderr_line --regexp "Usage of .*:"
 }
 
 @test "crowdsec (unknown flag)" {
     run -2 --separate-stderr timeout 2s "${CROWDSEC}" --foobar
-    run -0 echo "${stderr}"
-    assert_line "flag provided but not defined: -foobar"
-    assert_line --regexp "Usage of .*"
+    assert_stderr_line "flag provided but not defined: -foobar"
+    assert_stderr_line --regexp "Usage of .*"
 }
 
 @test "crowdsec (unknown argument)" {
     run -2 --separate-stderr timeout 2s "${CROWDSEC}" trololo
-    run -0 echo "${stderr}"
-    assert_line "argument provided but not defined: trololo"
-    assert_line --regexp "Usage of .*"
+    assert_stderr_line "argument provided but not defined: trololo"
+    assert_stderr_line --regexp "Usage of .*"
 }
 
 @test "crowdsec (no api and no agent)" {
     run -1 --separate-stderr timeout 2s "${CROWDSEC}" -no-api -no-cs
-    run -0 echo "${stderr}"
-    assert_line --partial "You must run at least the API Server or crowdsec"
+    assert_stderr_line --partial "You must run at least the API Server or crowdsec"
 }
 
 @test "crowdsec - print error on exit" {
@@ -60,12 +52,10 @@ declare stderr
     config_set '.db_config.type="meh"'
     run -1 --separate-stderr "${BIN_DIR}/crowdsec"
     refute_output
-    run -0 echo "${stderr}"
-    assert_output --partial "unable to create database client: unknown database type 'meh'"
+    assert_stderr --partial "unable to create database client: unknown database type 'meh'"
 }
 
 @test "CS_LAPI_SECRET not strong enough" {
     CS_LAPI_SECRET=foo run -1 --separate-stderr timeout 2s "${CROWDSEC}"
-    run -0 echo "${stderr}"
-    assert_output --partial "api server init: unable to run local API: controller init: CS_LAPI_SECRET not strong enough"
+    assert_stderr --partial "api server init: unable to run local API: controller init: CS_LAPI_SECRET not strong enough"
 }
diff --git a/tests/bats/02_nolapi.bats b/tests/bats/02_nolapi.bats
index 1b85e4806..a61275c89 100644
--- a/tests/bats/02_nolapi.bats
+++ b/tests/bats/02_nolapi.bats
@@ -21,8 +21,6 @@ teardown() {
     ./instance-crowdsec stop
 }
 
-declare stderr
-
 #----------
 
 @test "test without -no-api flag" {
@@ -40,17 +38,15 @@ declare stderr
     # really needs 4 secs on slow boxes
     run -1 --separate-stderr timeout 4s "${CROWDSEC}"
 
-    run -0 echo "${stderr}"
-    assert_output --partial "crowdsec local API is disabled"
+    assert_stderr --partial "crowdsec local API is disabled"
 }
 
 @test "capi status shouldn't be ok without api.server" {
     config_disable_lapi
     run -1 --separate-stderr cscli capi status
 
-    run -0 echo "${stderr}"
-    assert_output --partial "crowdsec local API is disabled"
-    assert_output --partial "There is no configuration on 'api.server:'"
+    assert_stderr --partial "crowdsec local API is disabled"
+    assert_stderr --partial "There is no configuration on 'api.server:'"
 }
 
 @test "cscli config show -o human" {
@@ -70,17 +66,15 @@ declare stderr
     run -1 --separate-stderr cscli config backup "${backupdir}"
     rm -rf -- "${backupdir:?}"
 
-    run -0 echo "${stderr}"
-    assert_output --partial "Failed to backup configurations"
-    assert_output --partial "file exists"
+    assert_stderr --partial "Failed to backup configurations"
+    assert_stderr --partial "file exists"
 }
 
 @test "lapi status shouldn't be ok without api.server" {
     config_disable_lapi
     ./instance-crowdsec start || true
     run -1 --separate-stderr cscli machines list
-    run -0 echo "${stderr}"
-    assert_output --partial "Local API is disabled, please run this command on the local API machine"
+    assert_stderr --partial "Local API is disabled, please run this command on the local API machine"
 }
 
 @test "cscli metrics" {
@@ -91,7 +85,6 @@ declare stderr
     assert_output --partial "ROUTE"
     assert_output --partial "/v1/watchers/login"
 
-    run -0 echo "${stderr}"
-    assert_output --partial "crowdsec local API is disabled"
-    assert_output --partial "Local API is disabled, please run this command on the local API machine"
+    assert_stderr --partial "crowdsec local API is disabled"
+    assert_stderr --partial "Local API is disabled, please run this command on the local API machine"
 }
diff --git a/tests/bats/03_noagent.bats b/tests/bats/03_noagent.bats
index 5d9b13077..060692073 100644
--- a/tests/bats/03_noagent.bats
+++ b/tests/bats/03_noagent.bats
@@ -20,8 +20,6 @@ teardown() {
     ./instance-crowdsec stop
 }
 
-declare stderr
-
 #----------
 
 @test "with agent: test without -no-cs flag" {
@@ -37,8 +35,7 @@ declare stderr
     config_disable_agent
     run -124 --separate-stderr timeout 2s "${CROWDSEC}"
 
-    run -0 echo "${stderr}"
-    assert_output --partial "crowdsec agent is disabled"
+    assert_stderr --partial "crowdsec agent is disabled"
 }
 
 @test "no agent: capi status should be ok" {
@@ -46,9 +43,7 @@ declare stderr
     config_enable_capi
     ./instance-crowdsec start
     run -0 --separate-stderr cscli capi status
-
-    run -0 echo "${stderr}"
-    assert_output --partial "You can successfully interact with Central API (CAPI)"
+    assert_stderr --partial "You can successfully interact with Central API (CAPI)"
 }
 
 @test "no agent: cscli config show" {
@@ -68,9 +63,8 @@ declare stderr
     assert_output --partial "Starting configuration backup"
     run -1 --separate-stderr cscli config backup "${backupdir}"
 
-    run -0 echo "${stderr}"
-    assert_output --partial "Failed to backup configurations"
-    assert_output --partial "file exists"
+    assert_stderr --partial "Failed to backup configurations"
+    assert_stderr --partial "file exists"
     rm -rf -- "${backupdir:?}"
 }
 
@@ -78,9 +72,7 @@ declare stderr
     config_disable_agent
     ./instance-crowdsec start
     run -0 --separate-stderr cscli lapi status
-
-    run -0 echo "${stderr}"
-    assert_output --partial "You can successfully interact with Local API (LAPI)"
+    assert_stderr --partial "You can successfully interact with Local API (LAPI)"
 }
 
 @test "cscli metrics" {
diff --git a/tests/bats/04_nocapi.bats b/tests/bats/04_nocapi.bats
index 0ca8ed475..c68cda8d5 100644
--- a/tests/bats/04_nocapi.bats
+++ b/tests/bats/04_nocapi.bats
@@ -20,35 +20,27 @@ teardown() {
     ./instance-crowdsec stop
 }
 
-declare stderr
-
 #----------
 
 @test "without capi: crowdsec LAPI should run without capi (-no-capi flag)" {
     config_set '.common.log_media="stdout"'
 
     run -124 --separate-stderr timeout 1s "${CROWDSEC}" -no-capi
-
-    run -0 echo "${stderr}"
-    assert_output --partial "Communication with CrowdSec Central API disabled from args"
+    assert_stderr --partial "Communication with CrowdSec Central API disabled from args"
 }
 
 @test "without capi: crowdsec LAPI should still work" {
     config_disable_capi
     run -124 --separate-stderr timeout 1s "${CROWDSEC}"
     # from `man timeout`: If  the  command  times  out,  and --preserve-status is not set, then exit with status 124.
-
-    run -0 echo "${stderr}"
-    assert_output --partial "push and pull to Central API disabled"
+    assert_stderr --partial "push and pull to Central API disabled"
 }
 
 @test "without capi: cscli capi status -> fail" {
     config_disable_capi
     ./instance-crowdsec start
     run -1 --separate-stderr cscli capi status
-
-    run -0 echo "${stderr}"
-    assert_output --partial "no configuration for Central API in "
+    assert_stderr --partial "no configuration for Central API in "
 }
 
 @test "no capi: cscli config show" {
@@ -66,10 +58,8 @@ declare stderr
     run -0 cscli config backup "${backupdir}"
     assert_output --partial "Starting configuration backup"
     run -1 --separate-stderr cscli config backup "${backupdir}"
-
-    run -0 echo "${stderr}"
-    assert_output --partial "Failed to backup configurations"
-    assert_output --partial "file exists"
+    assert_stderr --partial "Failed to backup configurations"
+    assert_stderr --partial "file exists"
     rm -rf -- "${backupdir:?}"
 }
 
@@ -77,9 +67,7 @@ declare stderr
     config_disable_capi
     ./instance-crowdsec start
     run -0 --separate-stderr cscli lapi status
-
-    run -0 echo "${stderr}"
-    assert_output --partial "You can successfully interact with Local API (LAPI)"
+    assert_stderr --partial "You can successfully interact with Local API (LAPI)"
 }
 
 @test "cscli metrics" {
@@ -90,6 +78,5 @@ declare stderr
     assert_output --partial "ROUTE"
     assert_output --partial '/v1/watchers/login'
 
-    run -0 echo "${stderr}"
-    assert_output --partial "Local Api Metrics:"
+    assert_stderr --partial "Local Api Metrics:"
 }
diff --git a/tests/bats/20_collections.bats b/tests/bats/20_collections.bats
index 0f05d1fe3..a4cb293e9 100644
--- a/tests/bats/20_collections.bats
+++ b/tests/bats/20_collections.bats
@@ -34,13 +34,28 @@ teardown() {
 }
 
 @test "can install a collection (as a regular user) and remove it" {
+    # collection is not installed
+    run -0 cscli collections list -o json
+    run -0 jq -r '.collections[].name' <(output)
+    refute_line "crowdsecurity/mysql"
+
+    # we install it
     run -0 cscli collections install crowdsecurity/mysql -o human
     assert_output --partial "Enabled crowdsecurity/mysql"
+
+    # it has been installed
     run -0 cscli collections list -o json
-    run -0 jq '.collections | length' <(output)
-    assert_output 3
+    run -0 jq -r '.collections[].name' <(output)
+    assert_line "crowdsecurity/mysql"
+
+    # we install it
     run -0 cscli collections remove crowdsecurity/mysql -o human
     assert_output --partial "Removed symlink [crowdsecurity/mysql]"
+
+    # it has been removed
+    run -0 cscli collections list -o json
+    run -0 jq -r '.collections[].name' <(output)
+    refute_line "crowdsecurity/mysql"
 }
 
 @test "cannot remove a collection twice" {
@@ -53,3 +68,5 @@ teardown() {
     assert_output --partial "unable to disable crowdsecurity/mysql"
     assert_output --partial "doesn't exist"
 }
+
+# TODO test download-only
diff --git a/tests/bats/72_plugin_badconfig.bats b/tests/bats/72_plugin_badconfig.bats
index 49d81df1a..49da10c4c 100644
--- a/tests/bats/72_plugin_badconfig.bats
+++ b/tests/bats/72_plugin_badconfig.bats
@@ -30,94 +30,81 @@ teardown() {
     chmod go-w "${PLUGIN_DIR}"/notification-http
 }
 
-declare stderr
-
 #----------
 
 @test "misconfigured plugin, only user is empty" {
     config_set '.plugin_config.user="" | .plugin_config.group="nogroup"'
     config_set "${PROFILES_PATH}" '.notifications=["http_default"]'
     run -1 --separate-stderr timeout 2s "${CROWDSEC}"
-    run -0 echo "${stderr}"
-    assert_output --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: both plugin user and group must be set"
+    assert_stderr --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: both plugin user and group must be set"
 }
 
 @test "misconfigured plugin, only group is empty" {
     config_set '(.plugin_config.user="nobody") | (.plugin_config.group="")'
     config_set "${PROFILES_PATH}" '.notifications=["http_default"]'
     run -1 --separate-stderr timeout 2s "${CROWDSEC}"
-    run -0 echo "${stderr}"
-    assert_output --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: both plugin user and group must be set"
+    assert_stderr --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: both plugin user and group must be set"
 }
 
 @test "misconfigured plugin, user does not exist" {
     config_set '(.plugin_config.user="userdoesnotexist") | (.plugin_config.group="groupdoesnotexist")'
     config_set "${PROFILES_PATH}" '.notifications=["http_default"]'
     run -1 --separate-stderr timeout 2s "${CROWDSEC}"
-    run -0 echo "${stderr}"
-    assert_output --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: user: unknown user userdoesnotexist"
+    assert_stderr --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: user: unknown user userdoesnotexist"
 }
 
 @test "misconfigured plugin, group does not exist" {
     config_set '(.plugin_config.user=strenv(USER)) | (.plugin_config.group="groupdoesnotexist")'
     config_set "${PROFILES_PATH}" '.notifications=["http_default"]'
     run -1 --separate-stderr timeout 2s "${CROWDSEC}"
-    run -0 echo "${stderr}"
-    assert_output --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: group: unknown group groupdoesnotexist"
+    assert_stderr --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: group: unknown group groupdoesnotexist"
 }
 
 @test "bad plugin name" {
     config_set "${PROFILES_PATH}" '.notifications=["http_default"]'
     cp "${PLUGIN_DIR}"/notification-http "${PLUGIN_DIR}"/badname
     run -1 --separate-stderr timeout 2s "${CROWDSEC}"
-    run -0 echo "${stderr}"
-    assert_output --partial "api server init: unable to run local API: while loading plugin: plugin name ${PLUGIN_DIR}/badname is invalid. Name should be like {type-name}"
+    assert_stderr --partial "api server init: unable to run local API: while loading plugin: plugin name ${PLUGIN_DIR}/badname is invalid. Name should be like {type-name}"
 }
 
 @test "bad plugin permission (group writable)" {
     config_set "${PROFILES_PATH}" '.notifications=["http_default"]'
     chmod g+w "${PLUGIN_DIR}"/notification-http
     run -1 --separate-stderr timeout 2s "${CROWDSEC}"
-    run -0 echo "${stderr}"
-    assert_output --partial "api server init: unable to run local API: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is group writable, group writable plugins are invalid"
+    assert_stderr --partial "api server init: unable to run local API: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is group writable, group writable plugins are invalid"
 }
 
 @test "bad plugin permission (world writable)" {
     config_set "${PROFILES_PATH}" '.notifications=["http_default"]'
     chmod o+w "${PLUGIN_DIR}"/notification-http
     run -1 --separate-stderr timeout 2s "${CROWDSEC}"
-    run -0 echo "${stderr}"
-    assert_output --partial "api server init: unable to run local API: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is world writable, world writable plugins are invalid"
+    assert_stderr --partial "api server init: unable to run local API: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is world writable, world writable plugins are invalid"
 }
 
 @test "config.yaml: missing .plugin_config section" {
     config_set 'del(.plugin_config)'
     config_set "${PROFILES_PATH}" '.notifications=["http_default"]'
     run -1 --separate-stderr timeout 2s "${CROWDSEC}"
-    run -0 echo "${stderr}"
-    assert_output --partial "api server init: plugins are enabled, but the plugin_config section is missing in the configuration"
+    assert_stderr --partial "api server init: plugins are enabled, but the plugin_config section is missing in the configuration"
 }
 
 @test "config.yaml: missing config_paths.notification_dir" {
     config_set 'del(.config_paths.notification_dir)'
     config_set "${PROFILES_PATH}" '.notifications=["http_default"]'
     run -1 --separate-stderr timeout 2s "${CROWDSEC}"
-    run -0 echo "${stderr}"
-    assert_output --partial "api server init: plugins are enabled, but config_paths.notification_dir is not defined"
+    assert_stderr --partial "api server init: plugins are enabled, but config_paths.notification_dir is not defined"
 }
 
 @test "config.yaml: missing config_paths.plugin_dir" {
     config_set 'del(.config_paths.plugin_dir)'
     config_set "${PROFILES_PATH}" '.notifications=["http_default"]'
     run -1 --separate-stderr timeout 2s "${CROWDSEC}"
-    run -0 echo "${stderr}"
-    assert_output --partial "api server init: plugins are enabled, but config_paths.plugin_dir is not defined"
+    assert_stderr --partial "api server init: plugins are enabled, but config_paths.plugin_dir is not defined"
 }
 
 @test "unable to run local API: while reading plugin config" {
     config_set '.config_paths.notification_dir="/this/path/does/not/exist"'
     config_set "${PROFILES_PATH}" '.notifications=["http_default"]'
     run -1 --separate-stderr timeout 2s "${CROWDSEC}"
-    run -0 echo "${stderr}"
-    assert_output --partial "api server init: unable to run local API: while loading plugin config: open /this/path/does/not/exist: no such file or directory"
+    assert_stderr --partial "api server init: unable to run local API: while loading plugin config: open /this/path/does/not/exist: no such file or directory"
 }
diff --git a/tests/bats/80_alerts.bats b/tests/bats/80_alerts.bats
index f17e712b1..d1d0f3f73 100644
--- a/tests/bats/80_alerts.bats
+++ b/tests/bats/80_alerts.bats
@@ -21,8 +21,6 @@ teardown() {
     ./instance-crowdsec stop
 }
 
-declare stderr
-
 #----------
 
 @test "cscli alerts list, with and without --machine" {
@@ -108,7 +106,7 @@ declare stderr
     run jq -c '.decisions[] | [.origin,.scenario,.scope,.simulated,.type,.value]' <<<"${alert}"
     assert_output --regexp "\[\"cscli\",\"manual 'ban' from 'githubciXXXXXXXXXXXXXXXXXXXXXXXX.*'\",\"Ip\",false,\"ban\",\"10.20.30.40\"\]"
     run jq -c '.source' <<<"${alert}"
-    assert_output '{"ip":"10.20.30.40","scope":"Ip","value":"10.20.30.40"}'
+    assert_json '{ip:"10.20.30.40",scope:"Ip",value:"10.20.30.40"}'
 }
 
 @test "no active alerts" {
@@ -124,8 +122,7 @@ declare stderr
 
 @test "cscli alerts delete" {
     run -0 --separate-stderr cscli alerts delete --all
-    run echo "${stderr}"
-    assert_output --partial 'alert(s) deleted'
+    assert_stderr --partial 'alert(s) deleted'
 
     # XXX TODO: delete by scope, id, value, scenario, range..
 }
diff --git a/tests/bats/90_decisions.bats b/tests/bats/90_decisions.bats
index bb2465eb8..1625b59d1 100644
--- a/tests/bats/90_decisions.bats
+++ b/tests/bats/90_decisions.bats
@@ -28,8 +28,7 @@ declare stderr
 @test "'decisions add' requires parameters" {
     run -1 --separate-stderr cscli decisions add
     assert_line "Usage:"
-    run echo "${stderr}"
-    assert_output --partial "Missing arguments, a value is required (--ip, --range or --scope and --value)"
+    assert_stderr --partial "Missing arguments, a value is required (--ip, --range or --scope and --value)"
 
     run -1 --separate-stderr cscli decisions add -o json
     run echo "${stderr}"
@@ -60,8 +59,7 @@ declare stderr
 
 @test "cscli decisions list, incorrect parameters" {
     run -1 --separate-stderr cscli decisions list --until toto
-    run echo "${stderr}"
-    assert_output --partial 'Unable to list decisions : performing request: API error: while parsing duration: time: invalid duration \"toto\"'
+    assert_stderr --partial 'Unable to list decisions : performing request: API error: while parsing duration: time: invalid duration \"toto\"'
     run -1 --separate-stderr cscli decisions list --until toto -o json
     run echo "${stderr}"
     run -0 jq -c '[.level, .msg]' <(output)
diff --git a/tests/lib/setup_file.sh b/tests/lib/setup_file.sh
index 2334bc174..27b2bca49 100755
--- a/tests/lib/setup_file.sh
+++ b/tests/lib/setup_file.sh
@@ -126,3 +126,53 @@ is_db_sqlite() {
 }
 export -f is_db_sqlite
 
+# compare ignoring the key order, and allow "expected" without quoted identifiers
+assert_json() {
+    # validate actual, sort
+    run -0 jq -Sen "${output}"
+    local actual="${output}"
+
+    # handle stdin, quote identifiers, sort
+    local expected="$1"
+    if [[ "${expected}" == "-" ]]; then
+        expected="$(cat)"
+    fi
+    run -0 jq -Sn "${expected}"
+    expected="${output}"
+
+    #shellcheck disable=SC2016
+    run jq -ne --argjson a "${actual}" --argjson b "${expected}" '$a == $b'
+    #shellcheck disable=SC2154
+    if [[ "${status}" -ne 0 ]]; then
+        echo "expect: $(jq -c <<<"${expected}")"
+        echo "actual: $(jq -c <<<"${actual}")"
+        diff <(echo "${actual}") <(echo "${expected}")
+        fail "json does not match"
+    fi
+}
+export -f assert_json
+
+assert_stderr() {
+    oldout="${output}"
+    run -0 echo "${stderr}"
+    assert_output "$@"
+    output="${oldout}"
+}
+export -f assert_stderr
+
+refute_stderr() {
+    oldout="${output}"
+    run -0 echo "${stderr}"
+    refute_output "$@"
+    output="${oldout}"
+}
+export -f refute_stderr
+
+assert_stderr_line() {
+    oldout="${output}"
+    run -0 echo "${stderr}"
+    assert_line "$@"
+    output="${oldout}"
+}
+export -f assert_stderr_line
+