From 899f3e7eb8e7f3738f2f33886e50f083e94f8fa6 Mon Sep 17 00:00:00 2001
From: mmetc <92726601+mmetc@users.noreply.github.com>
Date: Fri, 30 Dec 2022 15:57:24 +0100
Subject: [PATCH] func tests: redirect stderr to filter extra logs (#1961)

---
 tests/bats/01_base.bats                    |  8 +--
 tests/bats/04_capi.bats                    |  2 +-
 tests/bats/10_bouncers.bats                |  8 ++-
 tests/bats/20_collections.bats             | 16 ++---
 tests/bats/30_machines_tls.bats            |  8 +--
 tests/bats/40_cold-logs.bats               | 12 ++--
 tests/bats/40_live-ban.bats                |  2 +-
 tests/bats/50_simulation.bats              | 10 +--
 tests/bats/70_http_plugin.bats             |  8 +--
 tests/bats/71_dummy_plugin.bats            |  8 +--
 tests/bats/80_alerts.bats                  | 14 ++--
 tests/bats/90_decisions.bats               |  4 +-
 tests/bats/97_ipv4_single.bats             | 28 ++++----
 tests/bats/97_ipv6_single.bats             | 54 +++++++--------
 tests/bats/98_ipv4_range.bats              | 38 +++++------
 tests/bats/98_ipv6_range.bats              | 78 +++++++++++-----------
 tests/bats/99_lapi-stream-mode-scopes.bats |  8 +--
 tests/bats/99_lapi-stream-mode.bats        | 12 ++--
 tests/lib/setup_file.sh                    |  6 ++
 19 files changed, 166 insertions(+), 158 deletions(-)

diff --git a/tests/bats/01_base.bats b/tests/bats/01_base.bats
index 4ecf4fe16..9f37ccc6a 100644
--- a/tests/bats/01_base.bats
+++ b/tests/bats/01_base.bats
@@ -103,7 +103,7 @@ declare stderr
 }
 
 @test "cscli config show --key" {
-    run -0 cscli config show --key Config.API.Server.ListenURI
+    run -0 --separate-stderr cscli config show --key Config.API.Server.ListenURI
     assert_output "127.0.0.1:8080"
 }
 
@@ -238,7 +238,7 @@ declare stderr
     # we check for the presence of some objects. There may be others when we
     # use $PACKAGE_TESTING, so the order is not important.
 
-    run -0 cscli hub list -o human
+    run -0 --separate-stderr cscli hub list -o human
     assert_line --regexp '^ crowdsecurity/linux'
     assert_line --regexp '^ crowdsecurity/sshd'
     assert_line --regexp '^ crowdsecurity/dateparse-enrich'
@@ -248,7 +248,7 @@ declare stderr
     assert_line --regexp '^ crowdsecurity/ssh-bf'
     assert_line --regexp '^ crowdsecurity/ssh-slow-bf'
 
-    run -0 cscli hub list -o raw
+    run -0 --separate-stderr cscli hub list -o raw
     assert_line --regexp '^crowdsecurity/linux,enabled,[0-9]+\.[0-9]+,core linux support : syslog\+geoip\+ssh,collections$'
     assert_line --regexp '^crowdsecurity/sshd,enabled,[0-9]+\.[0-9]+,sshd support : parser and brute-force detection,collections$'
     assert_line --regexp '^crowdsecurity/dateparse-enrich,enabled,[0-9]+\.[0-9]+,,parsers$'
@@ -258,7 +258,7 @@ declare stderr
     assert_line --regexp '^crowdsecurity/ssh-bf,enabled,[0-9]+\.[0-9]+,Detect ssh bruteforce,scenarios$'
     assert_line --regexp '^crowdsecurity/ssh-slow-bf,enabled,[0-9]+\.[0-9]+,Detect slow ssh bruteforce,scenarios$'
 
-    run -0 cscli hub list -o json
+    run -0 --separate-stderr cscli hub list -o json
     run jq -r '.collections[].name, .parsers[].name, .scenarios[].name' <(output)
     assert_line 'crowdsecurity/linux'
     assert_line 'crowdsecurity/sshd'
diff --git a/tests/bats/04_capi.bats b/tests/bats/04_capi.bats
index 6564d1f6a..cc74abd91 100644
--- a/tests/bats/04_capi.bats
+++ b/tests/bats/04_capi.bats
@@ -37,7 +37,7 @@ setup() {
         [[ $(cscli alerts list -a -o json 2>/dev/null || cscli alerts list -o json) != "null" ]] && break
     done
 
-    run cscli alerts list -a -o json
+    run --separate-stderr cscli alerts list -a -o json
     if [[ "${status}" -ne 0 ]]; then
         run --separate-stderr cscli alerts list -o json
     fi
diff --git a/tests/bats/10_bouncers.bats b/tests/bats/10_bouncers.bats
index 832fea019..b22c5d861 100644
--- a/tests/bats/10_bouncers.bats
+++ b/tests/bats/10_bouncers.bats
@@ -40,12 +40,14 @@ teardown() {
     run -0 cscli bouncers add ciTestBouncer
     run -1 --separate-stderr cscli bouncers add ciTestBouncer -o json
 
-    run -0 jq -r '.level' <(stderr)
+    # XXX temporary hack to filter out unwanted log lines that may appear before
+    # log configuration (= not json)
+    run -0 jq -r '.level' <(stderr | grep "^{")
     assert_output 'fatal'
-    run -0 jq -r '.msg' <(stderr)
+    run -0 jq -r '.msg' <(stderr | grep "^{")
     assert_output "unable to create bouncer: bouncer ciTestBouncer already exists"
 
-    run -0 cscli bouncers list -o json
+    run -0 --separate-stderr cscli bouncers list -o json
     run -0 jq '. | length' <(output)
     assert_output 1
 }
diff --git a/tests/bats/20_collections.bats b/tests/bats/20_collections.bats
index c63d4c3bd..842a637ea 100644
--- a/tests/bats/20_collections.bats
+++ b/tests/bats/20_collections.bats
@@ -28,23 +28,23 @@ teardown() {
 }
 
 @test "there are 2 collections (linux and sshd)" {
-    run -0 cscli collections list -o json
+    run -0 --separate-stderr cscli collections list -o json
     run -0 jq '.collections | length' <(output)
     assert_output 2
 }
 
 @test "can install a collection (as a regular user) and remove it" {
     # collection is not installed
-    run -0 cscli collections list -o json
+    run -0 --separate-stderr cscli collections list -o json
     run -0 jq -r '.collections[].name' <(output)
     refute_line "crowdsecurity/mysql"
 
     # we install it
-    run -0 cscli collections install crowdsecurity/mysql -o human
-    assert_output --partial "Enabled crowdsecurity/mysql"
+    run -0 --separate-stderr cscli collections install crowdsecurity/mysql -o human
+    assert_stderr --partial "Enabled crowdsecurity/mysql"
 
     # it has been installed
-    run -0 cscli collections list -o json
+    run -0 --separate-stderr cscli collections list -o json
     run -0 jq -r '.collections[].name' <(output)
     assert_line "crowdsecurity/mysql"
 
@@ -53,7 +53,7 @@ teardown() {
     assert_output --partial "Removed symlink [crowdsecurity/mysql]"
 
     # it has been removed
-    run -0 cscli collections list -o json
+    run -0 --separate-stderr cscli collections list -o json
     run -0 jq -r '.collections[].name' <(output)
     refute_line "crowdsecurity/mysql"
 }
@@ -66,7 +66,7 @@ teardown() {
 
     run -0 --separate-stderr cscli collections remove crowdsecurity/sshd --force
     assert_stderr --partial "Removed symlink [crowdsecurity/sshd]"
-    run -0 cscli collections inspect crowdsecurity/linux -o json
+    run -0 --separate-stderr cscli collections inspect crowdsecurity/linux -o json
     run -0 jq -r '.tainted' <(output)
     assert_output "true"
 }
@@ -105,7 +105,7 @@ teardown() {
     run -0 cscli collections remove --all
     assert_output --partial "Removed symlink [crowdsecurity/sshd]"
     assert_output --partial "Removed symlink [crowdsecurity/linux]"
-    run -0 cscli hub list -o json
+    run -0 --separate-stderr cscli hub list -o json
     assert_json '{collections:[],parsers:[],postoverflows:[],scenarios:[]}'
     run -0 cscli collections remove --all
     assert_output --partial 'Disabled 0 items'
diff --git a/tests/bats/30_machines_tls.bats b/tests/bats/30_machines_tls.bats
index 05976c230..cd2a57936 100644
--- a/tests/bats/30_machines_tls.bats
+++ b/tests/bats/30_machines_tls.bats
@@ -80,7 +80,7 @@ teardown() {
 
     config_set "${CONFIG_DIR}/local_api_credentials.yaml" 'del(.login,.password)'
     ./instance-crowdsec start
-    run -0 cscli machines list -o json
+    run -0 --separate-stderr cscli machines list -o json
     assert_output '[]'
 }
 
@@ -95,7 +95,7 @@ teardown() {
     config_set "${CONFIG_DIR}/local_api_credentials.yaml" 'del(.login,.password)'
     ./instance-crowdsec start
     run -0 cscli lapi status
-    run -0 cscli machines list -o json
+    run -0 --separate-stderr cscli machines list -o json
     run -0 jq -c '[. | length, .[0].machineId[0:32], .[0].isValidated, .[0].ipAddress, .[0].auth_type]' <(output)
 
     assert_output '[1,"localhost@127.0.0.1",true,"127.0.0.1","tls"]'
@@ -111,7 +111,7 @@ teardown() {
     '
     config_set "${CONFIG_DIR}/local_api_credentials.yaml" 'del(.login,.password)'
     ./instance-crowdsec start
-    run -0 cscli machines list -o json
+    run -0 --separate-stderr cscli machines list -o json
     assert_output '[]'
 }
 
@@ -125,6 +125,6 @@ teardown() {
 
     config_set "${CONFIG_DIR}/local_api_credentials.yaml" 'del(.login,.password)'
     ./instance-crowdsec start
-    run -0 cscli machines list -o json
+    run -0 --separate-stderr cscli machines list -o json
     assert_output '[]'
 }
diff --git a/tests/bats/40_cold-logs.bats b/tests/bats/40_cold-logs.bats
index c04cf6a82..465dfb6eb 100644
--- a/tests/bats/40_cold-logs.bats
+++ b/tests/bats/40_cold-logs.bats
@@ -29,35 +29,35 @@ setup() {
 #----------
 
 @test "we have one decision" {
-    run -0 cscli decisions list -o json
+    run -0 --separate-stderr cscli decisions list -o json
     run -0 jq '. | length' <(output)
     assert_output 1
 }
 
 @test "1.1.1.172 has been banned" {
-    run -0 cscli decisions list -o json
+    run -0 --separate-stderr cscli decisions list -o json
     run -0 jq -r '.[].decisions[0].value' <(output)
     assert_output '1.1.1.172'
 }
 
 @test "1.1.1.172 has been banned (range/contained: -r 1.1.1.0/24 --contained)" {
-    run -0 cscli decisions list -r 1.1.1.0/24 --contained -o json
+    run -0 --separate-stderr cscli decisions list -r 1.1.1.0/24 --contained -o json
     run -0 jq -r '.[].decisions[0].value' <(output)
     assert_output '1.1.1.172'
 }
 
 @test "1.1.1.172 has not been banned (range/NOT-contained: -r 1.1.2.0/24)" {
-    run -0 cscli decisions list -r 1.1.2.0/24 -o json
+    run -0 --separate-stderr cscli decisions list -r 1.1.2.0/24 -o json
     assert_output 'null'
 }
 
 @test "1.1.1.172 has been banned (exact: -i 1.1.1.172)" {
-    run -0 cscli decisions list -i 1.1.1.172 -o json
+    run -0 --separate-stderr cscli decisions list -i 1.1.1.172 -o json
     run -0 jq -r '.[].decisions[0].value' <(output)
     assert_output '1.1.1.172'
 }
 
 @test "1.1.1.173 has not been banned (exact: -i 1.1.1.173)" {
-    run -0 cscli decisions list -i 1.1.1.173 -o json
+    run -0 --separate-stderr cscli decisions list -i 1.1.1.173 -o json
     assert_output 'null'
 }
diff --git a/tests/bats/40_live-ban.bats b/tests/bats/40_live-ban.bats
index 8551b28ce..bb54e6037 100644
--- a/tests/bats/40_live-ban.bats
+++ b/tests/bats/40_live-ban.bats
@@ -39,7 +39,7 @@ teardown() {
     fake_log >>"${tmpfile}"
     sleep 2
     rm -f -- "${tmpfile}"
-    run -0 cscli decisions list -o json
+    run -0 --separate-stderr cscli decisions list -o json
     run -0 jq -r '.[].decisions[0].value' <(output)
     assert_output '1.1.1.172'
 }
diff --git a/tests/bats/50_simulation.bats b/tests/bats/50_simulation.bats
index bff6ba61f..7fc2d7435 100644
--- a/tests/bats/50_simulation.bats
+++ b/tests/bats/50_simulation.bats
@@ -29,7 +29,7 @@ setup() {
 @test "we have one decision" {
     run -0 cscli simulation disable --global
     fake_log | "${CROWDSEC}" -dsn file:///dev/fd/0 -type syslog -no-api
-    run -0 cscli decisions list -o json
+    run -0 --separate-stderr cscli decisions list -o json
     run -0 jq '. | length' <(output)
     assert_output 1
 }
@@ -37,7 +37,7 @@ setup() {
 @test "1.1.1.174 has been banned (exact)" {
     run -0 cscli simulation disable --global
     fake_log | "${CROWDSEC}" -dsn file:///dev/fd/0 -type syslog -no-api
-    run -0 cscli decisions list -o json
+    run -0 --separate-stderr cscli decisions list -o json
     run -0 jq -r '.[].decisions[0].value' <(output)
     assert_output '1.1.1.174'
 }
@@ -45,7 +45,7 @@ setup() {
 @test "decision has simulated == false (exact)" {
     run -0 cscli simulation disable --global
     fake_log | "${CROWDSEC}" -dsn file:///dev/fd/0 -type syslog -no-api
-    run -0 cscli decisions list -o json
+    run -0 --separate-stderr cscli decisions list -o json
     run -0 jq '.[].decisions[0].simulated' <(output)
     assert_output 'false'
 }
@@ -53,7 +53,7 @@ setup() {
 @test "simulated scenario, listing non-simulated: expect no decision" {
     run -0 cscli simulation enable crowdsecurity/ssh-bf
     fake_log | "${CROWDSEC}" -dsn file:///dev/fd/0 -type syslog -no-api
-    run -0 cscli decisions list --no-simu -o json
+    run -0 --separate-stderr cscli decisions list --no-simu -o json
     assert_output 'null'
 }
 
@@ -61,6 +61,6 @@ setup() {
     run -0 cscli simulation disable crowdsecurity/ssh-bf
     run -0 cscli simulation enable --global
     fake_log | "${CROWDSEC}" -dsn file:///dev/fd/0 -type syslog -no-api
-    run -0 cscli decisions list --no-simu -o json
+    run -0 --separate-stderr cscli decisions list --no-simu -o json
     assert_output 'null'
 }
diff --git a/tests/bats/70_http_plugin.bats b/tests/bats/70_http_plugin.bats
index aeaa3fdbb..1b1b01399 100644
--- a/tests/bats/70_http_plugin.bats
+++ b/tests/bats/70_http_plugin.bats
@@ -54,11 +54,11 @@ setup() {
 #----------
 
 @test "add two bans" {
-    run -0 cscli decisions add --ip 1.2.3.4 --duration 30s
-    assert_output --partial 'Decision successfully added'
+    run -0 --separate-stderr cscli decisions add --ip 1.2.3.4 --duration 30s
+    assert_stderr --partial 'Decision successfully added'
 
-    run -0 cscli decisions add --ip 1.2.3.5 --duration 30s
-    assert_output --partial 'Decision successfully added'
+    run -0 --separate-stderr cscli decisions add --ip 1.2.3.5 --duration 30s
+    assert_stderr --partial 'Decision successfully added'
     sleep 5
 }
 
diff --git a/tests/bats/71_dummy_plugin.bats b/tests/bats/71_dummy_plugin.bats
index f5bdd0eb3..dd95295b0 100644
--- a/tests/bats/71_dummy_plugin.bats
+++ b/tests/bats/71_dummy_plugin.bats
@@ -58,11 +58,11 @@ setup() {
 #----------
 
 @test "add two bans" {
-    run -0 cscli decisions add --ip 1.2.3.4 --duration 30s
-    assert_output --partial 'Decision successfully added'
+    run -0 --separate-stderr cscli decisions add --ip 1.2.3.4 --duration 30s
+    assert_stderr --partial 'Decision successfully added'
 
-    run -0 cscli decisions add --ip 1.2.3.5 --duration 30s
-    assert_output --partial 'Decision successfully added'
+    run -0 --separate-stderr cscli decisions add --ip 1.2.3.5 --duration 30s
+    assert_stderr --partial 'Decision successfully added'
     sleep 2
 }
 
diff --git a/tests/bats/80_alerts.bats b/tests/bats/80_alerts.bats
index 520c65874..87eccf45a 100644
--- a/tests/bats/80_alerts.bats
+++ b/tests/bats/80_alerts.bats
@@ -52,7 +52,7 @@ teardown() {
     assert_output --regexp ".* ID .* value .* reason .* country .* as .* decisions .* created_at .*"
     assert_output --regexp ".*Ip:10.20.30.40.*manual 'ban' from.*ban:1.*"
 
-    run -0 cscli alerts list -o json
+    run -0 --separate-stderr cscli alerts list -o json
     run -0 jq -c '.[].decisions[0] | [.origin, .scenario, .scope, .simulated, .type, .value]' <(output)
     assert_line --regexp "\[\"cscli\",\"manual 'ban' from 'githubciXXXXXXXXXXXXXXXXXXXXXXXX([a-zA-Z0-9]{16})?'\",\"Ip\",false,\"ban\",\"10.20.30.40\"\]"
 
@@ -103,7 +103,7 @@ teardown() {
     assert_line --regexp "^ *type: ban$"
     assert_line --regexp "^ *value: 10.20.30.40$"
 
-    run -0 cscli alerts inspect "${ALERT_ID}" -o json
+    run -0 --separate-stderr cscli alerts inspect "${ALERT_ID}" -o json
     alert=${output}
     run jq -c '.decisions[] | [.origin,.scenario,.scope,.simulated,.type,.value]' <<<"${alert}"
     assert_output --regexp "\[\"cscli\",\"manual 'ban' from 'githubciXXXXXXXXXXXXXXXXXXXXXXXX.*'\",\"Ip\",false,\"ban\",\"10.20.30.40\"\]"
@@ -112,18 +112,18 @@ teardown() {
 }
 
 @test "no active alerts" {
-    run -0 cscli alerts list --until 200d -o human
+    run -0 --separate-stderr cscli alerts list --until 200d -o human
     assert_output "No active alerts"
-    run -0 cscli alerts list --until 200d -o json
+    run -0 --separate-stderr cscli alerts list --until 200d -o json
     assert_output "null"
-    run -0 cscli alerts list --until 200d -o raw
+    run -0 --separate-stderr cscli alerts list --until 200d -o raw
     assert_output "id,scope,value,reason,country,as,decisions,created_at"
-    run -0 cscli alerts list --until 200d -o raw --machine
+    run -0 --separate-stderr cscli alerts list --until 200d -o raw --machine
     assert_output "id,scope,value,reason,country,as,decisions,created_at,machine"
 }
 
 @test "cscli alerts delete (by id)" {
-    run -0 cscli alerts delete --help
+    run -0 --separate-stderr cscli alerts delete --help
     if [[ ! "$output" =~ "--id string" ]]; then
         skip "cscli alerts delete --id not supported"
     fi
diff --git a/tests/bats/90_decisions.bats b/tests/bats/90_decisions.bats
index bcf3ebb38..dbc955ce4 100644
--- a/tests/bats/90_decisions.bats
+++ b/tests/bats/90_decisions.bats
@@ -32,7 +32,7 @@ declare stderr
 
     run -1 --separate-stderr cscli decisions add -o json
     run echo "${stderr}"
-    run -0 jq -c '[ .level, .msg]' <(output)
+    run -0 jq -c '[ .level, .msg]' <(output | grep "^{")
     assert_output '["fatal","Missing arguments, a value is required (--ip, --range or --scope and --value)"]'
 }
 
@@ -62,6 +62,6 @@ declare stderr
     assert_stderr --partial 'Unable to list decisions : performing request: API error: while parsing duration: time: invalid duration \"toto\"'
     run -1 --separate-stderr cscli decisions list --until toto -o json
     run echo "${stderr}"
-    run -0 jq -c '[.level, .msg]' <(output)
+    run -0 jq -c '[.level, .msg]' <(output | grep "^{")
     assert_output '["fatal","Unable to list decisions : performing request: API error: while parsing duration: time: invalid duration \"toto\""]'
 }
diff --git a/tests/bats/97_ipv4_single.bats b/tests/bats/97_ipv4_single.bats
index b3ee93e99..18ff712f0 100644
--- a/tests/bats/97_ipv4_single.bats
+++ b/tests/bats/97_ipv4_single.bats
@@ -32,28 +32,28 @@ api() {
 @test "cli - first decisions list: must be empty" {
     # delete community pull
     run -0 cscli decisions delete --all
-    run -0 cscli decisions list -o json
+    run -0 --separate-stderr cscli decisions list -o json
     assert_output 'null'
 }
 
 @test "API - first decisions list: must be empty" {
-    run -0 api '/v1/decisions'
+    run -0 --separate-stderr api '/v1/decisions'
     assert_output 'null'
 }
 
 @test "adding decision for 1.2.3.4" {
-    run -0 cscli decisions add -i '1.2.3.4'
-    assert_output --partial 'Decision successfully added'
+    run -0 --separate-stderr cscli decisions add -i '1.2.3.4'
+    assert_stderr --partial 'Decision successfully added'
 }
 
 @test "CLI - all decisions" {
-    run -0 cscli decisions list -o json
+    run -0 --separate-stderr cscli decisions list -o json
     run -0 jq -r '.[0].decisions[0].value' <(output)
     assert_output '1.2.3.4'
 }
 
 @test "API - all decisions" {
-    run -0 api '/v1/decisions'
+    run -0 --separate-stderr api '/v1/decisions'
     run -0 jq -c '[ . | length, .[0].value ]' <(output)
     assert_output '[1,"1.2.3.4"]'
 }
@@ -61,47 +61,47 @@ api() {
 # check ip match
 
 @test "CLI - decision for 1.2.3.4" {
-    run -0 cscli decisions list -i '1.2.3.4' -o json
+    run -0 --separate-stderr cscli decisions list -i '1.2.3.4' -o json
     run -0 jq -r '.[0].decisions[0].value' <(output)
     assert_output '1.2.3.4'
 }
 
 @test "API - decision for 1.2.3.4" {
-    run -0 api '/v1/decisions?ip=1.2.3.4'
+    run -0 --separate-stderr api '/v1/decisions?ip=1.2.3.4'
     run -0 jq -r '.[0].value' <(output)
     assert_output '1.2.3.4'
 }
 
 @test "CLI - decision for 1.2.3.5" {
-    run -0 cscli decisions list -i '1.2.3.5' -o json
+    run -0 --separate-stderr cscli decisions list -i '1.2.3.5' -o json
     assert_output 'null'
 }
 
 @test "API - decision for 1.2.3.5" {
-    run -0 api '/v1/decisions?ip=1.2.3.5'
+    run -0 --separate-stderr api '/v1/decisions?ip=1.2.3.5'
     assert_output 'null'
 }
 
 ## check outer range match
 
 @test "CLI - decision for 1.2.3.0/24" {
-    run -0 cscli decisions list -r '1.2.3.0/24' -o json
+    run -0 --separate-stderr cscli decisions list -r '1.2.3.0/24' -o json
     assert_output 'null'
 }
 
 @test "API - decision for 1.2.3.0/24" {
-    run -0 api '/v1/decisions?range=1.2.3.0/24'
+    run -0 --separate-stderr api '/v1/decisions?range=1.2.3.0/24'
     assert_output 'null'
 }
 
 @test "CLI - decisions where IP in 1.2.3.0/24" {
-    run -0 cscli decisions list -r '1.2.3.0/24' --contained -o json
+    run -0 --separate-stderr cscli decisions list -r '1.2.3.0/24' --contained -o json
     run -0 jq -r '.[0].decisions[0].value' <(output)
     assert_output '1.2.3.4'
 }
 
 @test "API - decisions where IP in 1.2.3.0/24" {
-    run -0 api '/v1/decisions?range=1.2.3.0/24&contains=false'
+    run -0 --separate-stderr api '/v1/decisions?range=1.2.3.0/24&contains=false'
     run -0 jq -r '.[0].value' <(output)
     assert_output '1.2.3.4'
 }
diff --git a/tests/bats/97_ipv6_single.bats b/tests/bats/97_ipv6_single.bats
index e18099c2a..38ef681be 100644
--- a/tests/bats/97_ipv6_single.bats
+++ b/tests/bats/97_ipv6_single.bats
@@ -32,124 +32,124 @@ api() {
 @test "cli - first decisions list: must be empty" {
     # delete community pull
     run -0 cscli decisions delete --all
-    run -0 cscli decisions list -o json
+    run -0 --separate-stderr cscli decisions list -o json
     assert_output 'null'
 }
 
 @test "adding decision for ip 1111:2222:3333:4444:5555:6666:7777:8888" {
-    run -0 cscli decisions add -i '1111:2222:3333:4444:5555:6666:7777:8888'
-    assert_output --partial 'Decision successfully added'
+    run -0 --separate-stderr cscli decisions add -i '1111:2222:3333:4444:5555:6666:7777:8888'
+    assert_stderr --partial 'Decision successfully added'
 }
 
 @test "CLI - all decisions" {
-    run -0 cscli decisions list -o json
+    run -0 --separate-stderr cscli decisions list -o json
     run -0 jq -r '.[].decisions[0].value' <(output)
     assert_output '1111:2222:3333:4444:5555:6666:7777:8888'
 }
 
 @test "API - all decisions" {
-    run -0 api "/v1/decisions"
+    run -0 --separate-stderr api "/v1/decisions"
     run -0 jq -r '.[].value' <(output)
     assert_output '1111:2222:3333:4444:5555:6666:7777:8888'
 }
 
 @test "CLI - decisions for ip 1111:2222:3333:4444:5555:6666:7777:8888" {
-    run -0 cscli decisions list -i '1111:2222:3333:4444:5555:6666:7777:8888' -o json
+    run -0 --separate-stderr cscli decisions list -i '1111:2222:3333:4444:5555:6666:7777:8888' -o json
     run -0 jq -r '.[].decisions[0].value' <(output)
     assert_output '1111:2222:3333:4444:5555:6666:7777:8888'
 }
 
 @test "API - decisions for ip 1111:2222:3333:4444:5555:6666:7777:888" {
-    run -0 api '/v1/decisions?ip=1111:2222:3333:4444:5555:6666:7777:8888'
+    run -0 --separate-stderr api '/v1/decisions?ip=1111:2222:3333:4444:5555:6666:7777:8888'
     run -0 jq -r '.[].value' <(output)
     assert_output '1111:2222:3333:4444:5555:6666:7777:8888'
 }
 
 @test "CLI - decisions for ip 1211:2222:3333:4444:5555:6666:7777:8888" {
-    run -0 cscli decisions list -i '1211:2222:3333:4444:5555:6666:7777:8888' -o json
+    run -0 --separate-stderr cscli decisions list -i '1211:2222:3333:4444:5555:6666:7777:8888' -o json
     assert_output 'null'
 }
 
 @test "API - decisions for ip 1211:2222:3333:4444:5555:6666:7777:888" {
-    run -0 api '/v1/decisions?ip=1211:2222:3333:4444:5555:6666:7777:8888'
+    run -0 --separate-stderr api '/v1/decisions?ip=1211:2222:3333:4444:5555:6666:7777:8888'
     assert_output 'null'
 }
 
 @test "CLI - decisions for ip 1111:2222:3333:4444:5555:6666:7777:8887" {
-    run -0 cscli decisions list -i '1111:2222:3333:4444:5555:6666:7777:8887' -o json
+    run -0 --separate-stderr cscli decisions list -i '1111:2222:3333:4444:5555:6666:7777:8887' -o json
     assert_output 'null'
 }
 
 @test "API - decisions for ip 1111:2222:3333:4444:5555:6666:7777:8887" {
-    run -0 api '/v1/decisions?ip=1111:2222:3333:4444:5555:6666:7777:8887'
+    run -0 --separate-stderr api '/v1/decisions?ip=1111:2222:3333:4444:5555:6666:7777:8887'
     assert_output 'null'
 }
 
 @test "CLI - decisions for range 1111:2222:3333:4444:5555:6666:7777:8888/48" {
-    run -0 cscli decisions list -r '1111:2222:3333:4444:5555:6666:7777:8888/48' -o json
+    run -0 --separate-stderr cscli decisions list -r '1111:2222:3333:4444:5555:6666:7777:8888/48' -o json
     assert_output 'null'
 }
 
 @test "API - decisions for range 1111:2222:3333:4444:5555:6666:7777:8888/48" {
-    run -0 api '/v1/decisions?range=1111:2222:3333:4444:5555:6666:7777:8888/48'
+    run -0 --separate-stderr api '/v1/decisions?range=1111:2222:3333:4444:5555:6666:7777:8888/48'
     assert_output 'null'
 }
 
 @test "CLI - decisions for ip/range in 1111:2222:3333:4444:5555:6666:7777:8888/48" {
-    run -0 cscli decisions list -r '1111:2222:3333:4444:5555:6666:7777:8888/48' --contained -o json
+    run -0 --separate-stderr cscli decisions list -r '1111:2222:3333:4444:5555:6666:7777:8888/48' --contained -o json
     run -0 jq -r '.[].decisions[0].value' <(output)
     assert_output '1111:2222:3333:4444:5555:6666:7777:8888'
 }
 
 @test "API - decisions for ip/range in 1111:2222:3333:4444:5555:6666:7777:8888/48" {
-    run -0 api '/v1/decisions?range=1111:2222:3333:4444:5555:6666:7777:8888/48&&contains=false'
+    run -0 --separate-stderr api '/v1/decisions?range=1111:2222:3333:4444:5555:6666:7777:8888/48&&contains=false'
     run -0 jq -r '.[].value' <(output)
     assert_output '1111:2222:3333:4444:5555:6666:7777:8888'
 }
 
 @test "CLI - decisions for range 1111:2222:3333:4444:5555:6666:7777:8888/64" {
-    run -0 cscli decisions list -r '1111:2222:3333:4444:5555:6666:7777:8888/64' -o json
+    run -0 --separate-stderr cscli decisions list -r '1111:2222:3333:4444:5555:6666:7777:8888/64' -o json
     assert_output 'null'
 }
 
 @test "API - decisions for range 1111:2222:3333:4444:5555:6666:7777:8888/64" {
-    run -0 api '/v1/decisions?range=1111:2222:3333:4444:5555:6666:7777:8888/64'
+    run -0 --separate-stderr api '/v1/decisions?range=1111:2222:3333:4444:5555:6666:7777:8888/64'
     assert_output 'null'
 }
 
 @test "CLI - decisions for ip/range in 1111:2222:3333:4444:5555:6666:7777:8888/64" {
-    run -0 cscli decisions list -r '1111:2222:3333:4444:5555:6666:7777:8888/64' -o json --contained
+    run -0 --separate-stderr cscli decisions list -r '1111:2222:3333:4444:5555:6666:7777:8888/64' -o json --contained
     run -0 jq -r '.[].decisions[0].value' <(output)
     assert_output '1111:2222:3333:4444:5555:6666:7777:8888'
 }
 
 @test "API - decisions for ip/range in 1111:2222:3333:4444:5555:6666:7777:8888/64" {
-    run -0 api '/v1/decisions?range=1111:2222:3333:4444:5555:6666:7777:8888/64&&contains=false'
+    run -0 --separate-stderr api '/v1/decisions?range=1111:2222:3333:4444:5555:6666:7777:8888/64&&contains=false'
     run -0 jq -r '.[].value' <(output)
     assert_output '1111:2222:3333:4444:5555:6666:7777:8888'
 }
 
 @test "adding decision for ip 1111:2222:3333:4444:5555:6666:7777:8889" {
-    run -0 cscli decisions add -i '1111:2222:3333:4444:5555:6666:7777:8889'
-    assert_output --partial 'Decision successfully added'
+    run -0 --separate-stderr cscli decisions add -i '1111:2222:3333:4444:5555:6666:7777:8889'
+    assert_stderr --partial 'Decision successfully added'
 }
 
 @test "deleting decision for ip 1111:2222:3333:4444:5555:6666:7777:8889" {
-    run -0 cscli decisions delete -i '1111:2222:3333:4444:5555:6666:7777:8889'
-    assert_output --partial '1 decision(s) deleted'
+    run -0 --separate-stderr cscli decisions delete -i '1111:2222:3333:4444:5555:6666:7777:8889'
+    assert_stderr --partial '1 decision(s) deleted'
 }
 
 @test "CLI - decisions for ip 1111:2222:3333:4444:5555:6666:7777:8889 after delete" {
-    run -0 cscli decisions list -i '1111:2222:3333:4444:5555:6666:7777:8889' -o json
+    run -0 --separate-stderr cscli decisions list -i '1111:2222:3333:4444:5555:6666:7777:8889' -o json
     assert_output 'null'
 }
 
 @test "deleting decision for range 1111:2222:3333:4444:5555:6666:7777:8888/64" {
-    run -0 cscli decisions delete -r '1111:2222:3333:4444:5555:6666:7777:8888/64' --contained
-    assert_output --partial '1 decision(s) deleted'
+    run -0 --separate-stderr cscli decisions delete -r '1111:2222:3333:4444:5555:6666:7777:8888/64' --contained
+    assert_stderr --partial '1 decision(s) deleted'
 }
 
 @test "CLI - decisions for ip/range in 1111:2222:3333:4444:5555:6666:7777:8888/64 after delete" {
-    run -0 cscli decisions list -r '1111:2222:3333:4444:5555:6666:7777:8888/64' -o json --contained
+    run -0 --separate-stderr cscli decisions list -r '1111:2222:3333:4444:5555:6666:7777:8888/64' -o json --contained
     assert_output 'null'
 }
diff --git a/tests/bats/98_ipv4_range.bats b/tests/bats/98_ipv4_range.bats
index 38b237e7c..524b5944b 100644
--- a/tests/bats/98_ipv4_range.bats
+++ b/tests/bats/98_ipv4_range.bats
@@ -32,23 +32,23 @@ api() {
 @test "cli - first decisions list: must be empty" {
     # delete community pull
     run -0 cscli decisions delete --all
-    run -0 cscli decisions list -o json
+    run -0 --separate-stderr cscli decisions list -o json
     assert_output 'null'
 }
 
 @test "adding decision for range 4.4.4.0/24" {
-    run -0 cscli decisions add -r '4.4.4.0/24'
-    assert_output --partial 'Decision successfully added'
+    run -0 --separate-stderr cscli decisions add -r '4.4.4.0/24'
+    assert_stderr --partial 'Decision successfully added'
 }
 
 @test "CLI - all decisions" {
-    run -0 cscli decisions list -o json
+    run -0 --separate-stderr cscli decisions list -o json
     run -0 jq -r '.[0].decisions[0].value' <(output)
     assert_output '4.4.4.0/24'
 }
 
 @test "API - all decisions" {
-    run -0 api '/v1/decisions'
+    run -0 --separate-stderr api '/v1/decisions'
     run -0 jq -r '.[0].value' <(output)
     assert_output '4.4.4.0/24'
 }
@@ -56,55 +56,55 @@ api() {
 # check ip within/outside of range
 
 @test "CLI - decisions for ip 4.4.4." {
-    run -0 cscli decisions list -i '4.4.4.3' -o json
+    run -0 --separate-stderr cscli decisions list -i '4.4.4.3' -o json
     run -0 jq -r '.[0].decisions[0].value' <(output)
     assert_output '4.4.4.0/24'
 }
 
 @test "API - decisions for ip 4.4.4." {
-    run -0 api '/v1/decisions?ip=4.4.4.3'
+    run -0 --separate-stderr api '/v1/decisions?ip=4.4.4.3'
     run -0 jq -r '.[0].value' <(output)
     assert_output '4.4.4.0/24'
 }
 
 @test "CLI - decisions for ip contained in 4.4.4." {
-    run -0 cscli decisions list -i '4.4.4.4' -o json --contained
+    run -0 --separate-stderr cscli decisions list -i '4.4.4.4' -o json --contained
     assert_output 'null'
 }
 
 @test "API - decisions for ip contained in 4.4.4." {
-    run -0 api '/v1/decisions?ip=4.4.4.4&contains=false'
+    run -0 --separate-stderr api '/v1/decisions?ip=4.4.4.4&contains=false'
     assert_output 'null'
 }
 
 @test "CLI - decisions for ip 5.4.4." {
-    run -0 cscli decisions list -i '5.4.4.3' -o json
+    run -0 --separate-stderr cscli decisions list -i '5.4.4.3' -o json
     assert_output 'null'
 }
 
 @test "API - decisions for ip 5.4.4." {
-    run -0 api '/v1/decisions?ip=5.4.4.3'
+    run -0 --separate-stderr api '/v1/decisions?ip=5.4.4.3'
     assert_output 'null'
 }
 
 @test "CLI - decisions for range 4.4.0.0/1" {
-    run -0 cscli decisions list -r '4.4.0.0/16' -o json
+    run -0 --separate-stderr cscli decisions list -r '4.4.0.0/16' -o json
     assert_output 'null'
 }
 
 @test "API - decisions for range 4.4.0.0/1" {
-    run -0 api '/v1/decisions?range=4.4.0.0/16'
+    run -0 --separate-stderr api '/v1/decisions?range=4.4.0.0/16'
     assert_output 'null'
 }
 
 @test "CLI - decisions for ip/range in 4.4.0.0/1" {
-    run -0 cscli decisions list -r '4.4.0.0/16' -o json --contained
+    run -0 --separate-stderr cscli decisions list -r '4.4.0.0/16' -o json --contained
     run -0 jq -r '.[0].decisions[0].value' <(output)
     assert_output '4.4.4.0/24'
 }
 
 @test "API - decisions for ip/range in 4.4.0.0/1" {
-    run -0 api '/v1/decisions?range=4.4.0.0/16&contains=false'
+    run -0 --separate-stderr api '/v1/decisions?range=4.4.0.0/16&contains=false'
     run -0 jq -r '.[0].value' <(output)
     assert_output '4.4.4.0/24'
 }
@@ -112,23 +112,23 @@ api() {
 # check subrange
 
 @test "CLI - decisions for range 4.4.4.2/2" {
-    run -0 cscli decisions list -r '4.4.4.2/28' -o json
+    run -0 --separate-stderr cscli decisions list -r '4.4.4.2/28' -o json
     run -0 jq -r '.[].decisions[0].value' <(output)
     assert_output '4.4.4.0/24'
 }
 
 @test "API - decisions for range 4.4.4.2/2" {
-    run -0 api '/v1/decisions?range=4.4.4.2/28'
+    run -0 --separate-stderr api '/v1/decisions?range=4.4.4.2/28'
     run -0 jq -r '.[].value' <(output)
     assert_output '4.4.4.0/24'
 }
 
 @test "CLI - decisions for range 4.4.3.2/2" {
-    run -0 cscli decisions list -r '4.4.3.2/28' -o json
+    run -0 --separate-stderr cscli decisions list -r '4.4.3.2/28' -o json
     assert_output 'null'
 }
 
 @test "API - decisions for range 4.4.3.2/2" {
-    run -0 api '/v1/decisions?range=4.4.3.2/28'
+    run -0 --separate-stderr api '/v1/decisions?range=4.4.3.2/28'
     assert_output 'null'
 }
diff --git a/tests/bats/98_ipv6_range.bats b/tests/bats/98_ipv6_range.bats
index 857ae1aed..52074a816 100644
--- a/tests/bats/98_ipv6_range.bats
+++ b/tests/bats/98_ipv6_range.bats
@@ -32,23 +32,23 @@ api() {
 @test "cli - first decisions list: must be empty" {
     # delete community pull
     run -0 cscli decisions delete --all
-    run -0 cscli decisions list -o json
+    run -0 --separate-stderr cscli decisions list -o json
     assert_output 'null'
 }
 
 @test "adding decision for range aaaa:2222:3333:4444::/64" {
-    run -0 cscli decisions add -r 'aaaa:2222:3333:4444::/64'
-    assert_output --partial 'Decision successfully added'
+    run -0 --separate-stderr cscli decisions add -r 'aaaa:2222:3333:4444::/64'
+    assert_stderr --partial 'Decision successfully added'
 }
 
 @test "CLI - all decisions (2)" {
-    run -0 cscli decisions list -o json
+    run -0 --separate-stderr cscli decisions list -o json
     run -0 jq -r '.[].decisions[0].value' <(output)
     assert_output 'aaaa:2222:3333:4444::/64'
 }
 
 @test "API - all decisions (2)" {
-    run -0 api '/v1/decisions'
+    run -0 --separate-stderr api '/v1/decisions'
     run -0 jq -r '.[].value' <(output)
     assert_output 'aaaa:2222:3333:4444::/64'
 }
@@ -56,162 +56,162 @@ api() {
 # check ip within/out of range
 
 @test "CLI - decisions for ip aaaa:2222:3333:4444:5555:6666:7777:8888" {
-    run -0 cscli decisions list -i 'aaaa:2222:3333:4444:5555:6666:7777:8888' -o json
+    run -0 --separate-stderr cscli decisions list -i 'aaaa:2222:3333:4444:5555:6666:7777:8888' -o json
     run -0 jq -r '.[].decisions[0].value' <(output)
     assert_output 'aaaa:2222:3333:4444::/64'
 }
 
 @test "API - decisions for ip aaaa:2222:3333:4444:5555:6666:7777:8888" {
-    run -0 api '/v1/decisions?ip=aaaa:2222:3333:4444:5555:6666:7777:8888'
+    run -0 --separate-stderr api '/v1/decisions?ip=aaaa:2222:3333:4444:5555:6666:7777:8888'
     run -0 jq -r '.[].value' <(output)
     assert_output 'aaaa:2222:3333:4444::/64'
 }
 
 @test "CLI - decisions for ip aaaa:2222:3333:4445:5555:6666:7777:8888" {
-    run -0 cscli decisions list -i 'aaaa:2222:3333:4445:5555:6666:7777:8888' -o json
+    run -0 --separate-stderr cscli decisions list -i 'aaaa:2222:3333:4445:5555:6666:7777:8888' -o json
     assert_output 'null'
 }
 
 @test "API - decisions for ip aaaa:2222:3333:4445:5555:6666:7777:8888" {
-    run -0 api '/v1/decisions?ip=aaaa:2222:3333:4445:5555:6666:7777:8888'
+    run -0 --separate-stderr api '/v1/decisions?ip=aaaa:2222:3333:4445:5555:6666:7777:8888'
     assert_output 'null'
 }
 
 @test "CLI - decisions for ip aaa1:2222:3333:4444:5555:6666:7777:8887" {
-    run -0 cscli decisions list -i 'aaa1:2222:3333:4444:5555:6666:7777:8887' -o json
+    run -0 --separate-stderr cscli decisions list -i 'aaa1:2222:3333:4444:5555:6666:7777:8887' -o json
     assert_output 'null'
 }
 
 @test "API - decisions for ip aaa1:2222:3333:4444:5555:6666:7777:8887" {
-    run -0 api '/v1/decisions?ip=aaa1:2222:3333:4444:5555:6666:7777:8887'
+    run -0 --separate-stderr api '/v1/decisions?ip=aaa1:2222:3333:4444:5555:6666:7777:8887'
     assert_output 'null'
 }
 
 # check subrange within/out of range
 
 @test "CLI - decisions for range aaaa:2222:3333:4444:5555::/80" {
-    run -0 cscli decisions list -r 'aaaa:2222:3333:4444:5555::/80' -o json
+    run -0 --separate-stderr cscli decisions list -r 'aaaa:2222:3333:4444:5555::/80' -o json
     run -0 jq -r '.[].decisions[0].value' <(output)
     assert_output 'aaaa:2222:3333:4444::/64'
 }
 
 @test "API - decisions for range aaaa:2222:3333:4444:5555::/80" {
-    run -0 api '/v1/decisions?range=aaaa:2222:3333:4444:5555::/80'
+    run -0 --separate-stderr api '/v1/decisions?range=aaaa:2222:3333:4444:5555::/80'
     run -0 jq -r '.[].value' <(output)
     assert_output 'aaaa:2222:3333:4444::/64'
 }
 
 @test "CLI - decisions for range aaaa:2222:3333:4441:5555::/80" {
-    run -0 cscli decisions list -r 'aaaa:2222:3333:4441:5555::/80' -o json
+    run -0 --separate-stderr cscli decisions list -r 'aaaa:2222:3333:4441:5555::/80' -o json
     assert_output 'null'
 
 }
 
 @test "API - decisions for range aaaa:2222:3333:4441:5555::/80" {
-    run -0 api '/v1/decisions?range=aaaa:2222:3333:4441:5555::/80'
+    run -0 --separate-stderr api '/v1/decisions?range=aaaa:2222:3333:4441:5555::/80'
     assert_output 'null'
 }
 
 @test "CLI - decisions for range aaa1:2222:3333:4444:5555::/80" {
-    run -0 cscli decisions list -r 'aaa1:2222:3333:4444:5555::/80' -o json
+    run -0 --separate-stderr cscli decisions list -r 'aaa1:2222:3333:4444:5555::/80' -o json
     assert_output 'null'
 }
 
 @test "API - decisions for range aaa1:2222:3333:4444:5555::/80" {
-    run -0 api '/v1/decisions?range=aaa1:2222:3333:4444:5555::/80'
+    run -0 --separate-stderr api '/v1/decisions?range=aaa1:2222:3333:4444:5555::/80'
     assert_output 'null'
 }
 
 # check outer range
 
 @test "CLI - decisions for range aaaa:2222:3333:4444:5555:6666:7777:8888/48" {
-    run -0 cscli decisions list -r 'aaaa:2222:3333:4444:5555:6666:7777:8888/48' -o json
+    run -0 --separate-stderr cscli decisions list -r 'aaaa:2222:3333:4444:5555:6666:7777:8888/48' -o json
     assert_output 'null'
 }
 
 @test "API - decisions for range aaaa:2222:3333:4444:5555:6666:7777:8888/48" {
-    run -0 api '/v1/decisions?range=aaaa:2222:3333:4444:5555:6666:7777:8888/48'
+    run -0 --separate-stderr api '/v1/decisions?range=aaaa:2222:3333:4444:5555:6666:7777:8888/48'
     assert_output 'null'
 }
 
 @test "CLI - decisions for ip/range in aaaa:2222:3333:4444:5555:6666:7777:8888/48" {
-    run -0 cscli decisions list -r 'aaaa:2222:3333:4444:5555:6666:7777:8888/48' -o json --contained
+    run -0 --separate-stderr cscli decisions list -r 'aaaa:2222:3333:4444:5555:6666:7777:8888/48' -o json --contained
     run -0 jq -r '.[].decisions[0].value' <(output)
     assert_output 'aaaa:2222:3333:4444::/64'
 }
 
 @test "API - decisions for ip/range in aaaa:2222:3333:4444:5555:6666:7777:8888/48" {
-    run -0 api '/v1/decisions?range=aaaa:2222:3333:4444:5555:6666:7777:8888/48&contains=false'
+    run -0 --separate-stderr api '/v1/decisions?range=aaaa:2222:3333:4444:5555:6666:7777:8888/48&contains=false'
     run -0 jq -r '.[].value' <(output)
     assert_output 'aaaa:2222:3333:4444::/64'
 }
 
 @test "CLI - decisions for ip/range in aaaa:2222:3333:4445:5555:6666:7777:8888/48" {
-    run -0 cscli decisions list -r 'aaaa:2222:3333:4445:5555:6666:7777:8888/48' -o json
+    run -0 --separate-stderr cscli decisions list -r 'aaaa:2222:3333:4445:5555:6666:7777:8888/48' -o json
     assert_output 'null'
 }
 
 @test "API - decisions for ip/range in aaaa:2222:3333:4445:5555:6666:7777:8888/48" {
-    run -0 api '/v1/decisions?range=aaaa:2222:3333:4445:5555:6666:7777:8888/48'
+    run -0 --separate-stderr api '/v1/decisions?range=aaaa:2222:3333:4445:5555:6666:7777:8888/48'
     assert_output 'null'
 }
 
 # bbbb:db8:: -> bbbb:db8:0000:0000:0000:7fff:ffff:ffff
 
 @test "adding decision for range bbbb:db8::/81" {
-    run -0 cscli decisions add -r 'bbbb:db8::/81'
-    assert_output --partial 'Decision successfully added'
+    run -0 --separate-stderr cscli decisions add -r 'bbbb:db8::/81'
+    assert_stderr --partial 'Decision successfully added'
 }
 
 @test "CLI - decisions for ip bbbb:db8:0000:0000:0000:6fff:ffff:ffff" {
-    run -0 cscli decisions list -o json -i 'bbbb:db8:0000:0000:0000:6fff:ffff:ffff'
+    run -0 --separate-stderr cscli decisions list -o json -i 'bbbb:db8:0000:0000:0000:6fff:ffff:ffff'
     run -0 jq -r '.[].decisions[0].value' <(output)
     assert_output 'bbbb:db8::/81'
 }
 
 @test "API - decisions for ip in bbbb:db8:0000:0000:0000:6fff:ffff:ffff" {
-    run -0 api '/v1/decisions?ip=bbbb:db8:0000:0000:0000:6fff:ffff:ffff'
+    run -0 --separate-stderr api '/v1/decisions?ip=bbbb:db8:0000:0000:0000:6fff:ffff:ffff'
     run -0 jq -r '.[].value' <(output)
     assert_output 'bbbb:db8::/81'
 }
 
 @test "CLI - decisions for ip bbbb:db8:0000:0000:0000:8fff:ffff:ffff" {
-    run -0 cscli decisions list -o json -i 'bbbb:db8:0000:0000:0000:8fff:ffff:ffff'
+    run -0 --separate-stderr cscli decisions list -o json -i 'bbbb:db8:0000:0000:0000:8fff:ffff:ffff'
     assert_output 'null'
 }
 
 @test "API - decisions for ip in bbbb:db8:0000:0000:0000:8fff:ffff:ffff" {
-    run -0 api '/v1/decisions?ip=bbbb:db8:0000:0000:0000:8fff:ffff:ffff'
+    run -0 --separate-stderr api '/v1/decisions?ip=bbbb:db8:0000:0000:0000:8fff:ffff:ffff'
     assert_output 'null'
 }
 
 @test "deleting decision for range aaaa:2222:3333:4444:5555:6666:7777:8888/48" {
-    run -0 cscli decisions delete -r 'aaaa:2222:3333:4444:5555:6666:7777:8888/48' --contained
-    assert_output --partial '1 decision(s) deleted'
+    run -0 --separate-stderr cscli decisions delete -r 'aaaa:2222:3333:4444:5555:6666:7777:8888/48' --contained
+    assert_stderr --partial '1 decision(s) deleted'
 }
 
 @test "CLI - decisions for range aaaa:2222:3333:4444::/64 after delete" {
-    run -0 cscli decisions list -o json -r 'aaaa:2222:3333:4444::/64'
+    run -0 --separate-stderr cscli decisions list -o json -r 'aaaa:2222:3333:4444::/64'
     assert_output 'null'
 }
 
 @test "adding decision for ip bbbb:db8:0000:0000:0000:8fff:ffff:ffff" {
-    run -0 cscli decisions add -i 'bbbb:db8:0000:0000:0000:8fff:ffff:ffff'
-    assert_output --partial 'Decision successfully added'
+    run -0 --separate-stderr cscli decisions add -i 'bbbb:db8:0000:0000:0000:8fff:ffff:ffff'
+    assert_stderr --partial 'Decision successfully added'
 }
 
 @test "adding decision for ip bbbb:db8:0000:0000:0000:6fff:ffff:ffff" {
-    run -0 cscli decisions add -i 'bbbb:db8:0000:0000:0000:6fff:ffff:ffff'
-    assert_output --partial 'Decision successfully added'
+    run -0 --separate-stderr cscli decisions add -i 'bbbb:db8:0000:0000:0000:6fff:ffff:ffff'
+    assert_stderr --partial 'Decision successfully added'
 }
 
 @test "deleting decisions for range bbbb:db8::/81" {
-    run -0 cscli decisions delete -r 'bbbb:db8::/81' --contained
-    assert_output --partial '2 decision(s) deleted'
+    run -0 --separate-stderr cscli decisions delete -r 'bbbb:db8::/81' --contained
+    assert_stderr --partial '2 decision(s) deleted'
 }
 
 @test "CLI - all decisions (3)" {
-    run -0 cscli decisions list -o json
+    run -0 --separate-stderr cscli decisions list -o json
     run -0 jq -r '.[].decisions[0].value' <(output)
     assert_output 'bbbb:db8:0000:0000:0000:8fff:ffff:ffff'
 }
diff --git a/tests/bats/99_lapi-stream-mode-scopes.bats b/tests/bats/99_lapi-stream-mode-scopes.bats
index 60dd2e679..6e4bca021 100644
--- a/tests/bats/99_lapi-stream-mode-scopes.bats
+++ b/tests/bats/99_lapi-stream-mode-scopes.bats
@@ -29,10 +29,10 @@ api() {
 }
 
 @test "adding decisions for multiple scopes" {
-    run -0 cscli decisions add -i '1.2.3.6'
-    assert_output --partial 'Decision successfully added'
-    run -0 cscli decisions add --scope user --value toto
-    assert_output --partial 'Decision successfully added'
+    run -0 --separate-stderr cscli decisions add -i '1.2.3.6'
+    assert_stderr --partial 'Decision successfully added'
+    run -0 --separate-stderr cscli decisions add --scope user --value toto
+    assert_stderr --partial 'Decision successfully added'
 }
 
 @test "stream start (implicit ip scope)" {
diff --git a/tests/bats/99_lapi-stream-mode.bats b/tests/bats/99_lapi-stream-mode.bats
index 082dbafb5..0ee0a64b7 100644
--- a/tests/bats/99_lapi-stream-mode.bats
+++ b/tests/bats/99_lapi-stream-mode.bats
@@ -31,12 +31,12 @@ api() {
 @test "adding decisions for multiple ips" {
     run -0 cscli decisions add -i '1111:2222:3333:4444:5555:6666:7777:8888'
     run -0 cscli decisions add -i '1.2.3.4'
-    run -0 cscli decisions add -r '1.2.4.0/24'
-    assert_output --partial 'Decision successfully added'
+    run -0 --separate-stderr cscli decisions add -r '1.2.4.0/24'
+    assert_stderr --partial 'Decision successfully added'
 }
 
 @test "stream start" {
-    run -0 api "/v1/decisions/stream?startup=true"
+    run -0 --separate-stderr api "/v1/decisions/stream?startup=true"
     if is_db_mysql; then sleep 3; fi
     run -0 jq -r '.new' <(output)
     assert_output --partial '1111:2222:3333:4444:5555:6666:7777:8888'
@@ -47,7 +47,7 @@ api() {
 @test "stream cont (add)" {
     run -0 cscli decisions add -i '1.2.3.5'
     if is_db_mysql; then sleep 3; fi
-    run -0 api "/v1/decisions/stream"
+    run -0 --separate-stderr api "/v1/decisions/stream"
     run -0 jq -r '.new' <(output)
     assert_output --partial '1.2.3.5'
 }
@@ -55,13 +55,13 @@ api() {
 @test "stream cont (del)" {
     run -0 cscli decisions delete -i '1.2.3.4'
     if is_db_mysql; then sleep 3; fi
-    run -0 api "/v1/decisions/stream"
+    run -0 --separate-stderr api "/v1/decisions/stream"
     run -0 jq -r '.deleted' <(output)
     assert_output --partial '1.2.3.4'
 }
 
 @test "stream restart" {
-    run -0 api "/v1/decisions/stream?startup=true"
+    run -0 --separate-stderr api "/v1/decisions/stream?startup=true"
     api_out=${output}
     run -0 jq -r '.deleted' <(output)
     assert_output --partial '1.2.3.4'
diff --git a/tests/lib/setup_file.sh b/tests/lib/setup_file.sh
index 7e9401b6f..25842fcdc 100755
--- a/tests/lib/setup_file.sh
+++ b/tests/lib/setup_file.sh
@@ -211,3 +211,9 @@ plaintext() {
     sed -E 's/\x1B\[[0-9;]*[JKmsu]//g'
 }
 export -f plaintext
+
+# like run but defaults to separate stderr and stdout
+rune() {
+    run --separate-stderr "$@"
+}
+export -f rune