From b792c459218a2d0697c8d395df6b84e372a18a2c Mon Sep 17 00:00:00 2001 From: Manuel Sabban Date: Fri, 18 Mar 2022 17:36:51 +0100 Subject: [PATCH] Mysql enabled tests (#1372) * add msyql backend test Co-authored-by: sabban <15465465+sabban@users.noreply.github.com> --- .github/workflows/ci_bats_mysql.yaml | 58 ++++++++++++++ tests/bats.mk | 2 + tests/fixtures/mysql | 109 +++++++++++++++++++++++++++ tests/instance-data | 5 ++ tests/instance-db | 13 +++- tests/instance-mysql | 56 ++++++++++++++ tests/instance-sqlite | 7 +- 7 files changed, 248 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/ci_bats_mysql.yaml create mode 100755 tests/fixtures/mysql create mode 100755 tests/instance-mysql diff --git a/.github/workflows/ci_bats_mysql.yaml b/.github/workflows/ci_bats_mysql.yaml new file mode 100644 index 000000000..e7dbde028 --- /dev/null +++ b/.github/workflows/ci_bats_mysql.yaml @@ -0,0 +1,58 @@ +name: BATS functional tests with MYSQL + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + build: + name: "Build the application" + runs-on: ubuntu-latest + timeout-minutes: 20 + services: + mariadb: + image: mysql:latest + env: + MYSQL_ROOT_PASSWORD: mysql + ports: + - 3306:3306 + steps: + + - name: "Set up Go 1.17" + uses: actions/setup-go@v1 + with: + go-version: 1.17 + id: go + + - name: "Clone CrowdSec" + uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: true + + - name: "Install bats dependencies" + run: | + sudo apt install daemonize netcat-openbsd + GO111MODULE=on go get github.com/mikefarah/yq/v4 + + - name: "BATS: build crowdsec" + run: make bats-clean bats-build + env: + DB_BACKEND: mysql + MYSQL_HOST: 127.0.0.1 + MYSQL_PORT: 3306 + MYSQL_PASSWORD: mysql + MYSQL_USER: root + + - name: "BATS: run tests" + run: make bats-test + env: + DB_BACKEND: mysql + MYSQL_HOST: 127.0.0.1 + MYSQL_PORT: 3306 + MYSQL_PASSWORD: mysql + MYSQL_USER: root diff --git a/tests/bats.mk b/tests/bats.mk index 7fd1fb253..9380d71b8 100644 --- a/tests/bats.mk +++ b/tests/bats.mk @@ -13,6 +13,7 @@ LOCAL_INIT_DIR = $(TEST_DIR)/local-init LOG_DIR = $(LOCAL_DIR)/var/log PID_DIR = $(LOCAL_DIR)/var/run PLUGIN_DIR = $(LOCAL_DIR)/lib/crowdsec/plugins +DB_BACKEND := $(or $(DB_BACKEND), "sqlite") define ENV := export TEST_DIR="$(TEST_DIR)" @@ -24,6 +25,7 @@ export LOCAL_INIT_DIR="$(LOCAL_INIT_DIR)" export LOG_DIR="$(LOG_DIR)" export PID_DIR="$(PID_DIR)" export PLUGIN_DIR="$(PLUGIN_DIR)" +export DB_BACKEND="$(DB_BACKEND)" endef bats-all: bats-clean bats-build bats-test diff --git a/tests/fixtures/mysql b/tests/fixtures/mysql new file mode 100755 index 000000000..83269bc61 --- /dev/null +++ b/tests/fixtures/mysql @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +set -eu +script_name=$0 + +die() { + echo >&2 "$@" + exit 1 +} + +MYSQL_HOST=${MYSQL_HOST:-127.0.0.1} +MYSQL_PORT=${MYSQL_PORT:-3306} +MYSQL_PASSWORD=${MYSQL_PASSWORD:-password} +MYSQL_USER=${MYSQL_USER:-root} + +about() { + die "usage: $0 [ setup | cleanup | configure | backup | restore]" +} + +check_mysql_client() { + if ! command -v mysql >/dev/null; then + die "missing required program 'mysql' as a mysql client (package mariadb-client-core-10.6 on debian like system)" + fi +} + +send_mysql() { + if [ "$#" -lt 1 ]; then + die "missing requirement argument to send the mysql database" + fi + + /bin/echo "$1" | mysql -h ${MYSQL_HOST} -u ${MYSQL_USER} --port=${MYSQL_PORT} -p${MYSQL_PASSWORD} +} + +requirements() { + check_mysql_client +} + +setup_database() { + send_mysql "CREATE DATABASE IF NOT EXISTS crowdsec;" + send_mysql "CREATE USER IF NOT EXISTS 'crowdsec' IDENTIFIED BY 'crowdsec';" + send_mysql "GRANT ALL PRIVILEGES ON crowdsec.* TO 'crowdsec';" +} + +# cleanup is idempotent +cleanup_database() { + send_mysql "DROP DATABASE IF EXISTS crowdsec;" + send_mysql "DROP USER IF EXISTS crowdsec;" +} + +backup_database() { + if [ "$#" -lt 1 ]; then + die "missing file to backup database to" + fi + mysqldump -h${MYSQL_HOST} -u${MYSQL_USER} -p${MYSQL_PASSWORD} --databases crowdsec > $1 +} + +restore_database() { + if [ "$#" -lt 1 ]; then + die "missing file to restore database to" + fi + + if [ -f $1 ]; then + mysql -h ${MYSQL_HOST} -u ${MYSQL_USER} --port=${MYSQL_PORT} -p${MYSQL_PASSWORD} < $1 + else + die "$2 doesn\'t exist" + fi + send_mysql "CREATE USER IF NOT EXISTS 'crowdsec' IDENTIFIED BY 'crowdsec';" + send_mysql "GRANT ALL PRIVILEGES ON crowdsec.* TO 'crowdsec';" +} + +setup_configuration() { + MYSQL_PORT=${MYSQL_PORT} MYSQL_HOST=${MYSQL_HOST} yq ' + .db_config.type="mysql"| + .db_config.user="crowdsec" | + .db_config.password="crowdsec" | + .db_config.db_name="crowdsec" | + .db_config.host=strenv(MYSQL_HOST) | + .db_config.port=env(MYSQL_PORT) | + del(.db_config.db_path) + ' -i "${CONFIG_YAML}" +} + + +case "$1" in + setup) + setup_database + ;; + cleanup) + cleanup_database + ;; + configure) + setup_configuration + ;; + backup) + if [ "$#" -lt 2 ]; then + die "missing file to restore backup to" + fi + backup_database $2 + ;; + restore) + if [ "$#" -lt 2 ]; then + die "missing file to restore restore to" + fi + restore_database $2 + ;; + *) + about + ;; +esac; diff --git a/tests/instance-data b/tests/instance-data index eb24acf39..7168c382c 100755 --- a/tests/instance-data +++ b/tests/instance-data @@ -25,6 +25,7 @@ cd "${THIS_DIR}" [ -z "${CSCLI-}" ] && die "\$CSCLI must be defined." [ -z "${LOCAL_INIT_DIR-}" ] && die "\$LOCAL_INIT_DIR must be defined." [ -z "${PLUGIN_DIR-}" ] && die "\$PLUGIN_DIR must be defined." +[ -z "${DB_BACKEND-}" ] && die "\$DB_BACKEND must be defined." if [ ! -f "${CSCLI}" ]; then die "${CSCLI} is missing. Please build (with 'make bats-build') or install it." @@ -40,6 +41,8 @@ export CONFIG_DIR remove_init_data() { rm -rf -- "${LOCAL_DIR:?}/${REL_CONFIG_DIR}"/* "${LOCAL_DIR:?}/${REL_DATA_DIR:?}"/* + + ./instance-db cleanup } config_generate() { @@ -82,6 +85,8 @@ make_init_data() { mkdir -p "${CONFIG_DIR}/notifications" config_generate + ./instance-db configure + ./instance-db setup mkdir -p "${CONFIG_DIR}/hub" "${CSCLI}" machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto diff --git a/tests/instance-db b/tests/instance-db index 3bc7d2537..c0991ce62 100755 --- a/tests/instance-db +++ b/tests/instance-db @@ -4,4 +4,15 @@ THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) cd "${THIS_DIR}" -./instance-sqlite "$@" +case "$DB_BACKEND" in + sqlite) + ./instance-sqlite "$@" + ;; + mysql) + ./instance-mysql "$@" + ;; + *) + echo >&2 "unknown $DB_BACKEND" + exit 1 + ;; +esac; diff --git a/tests/instance-mysql b/tests/instance-mysql new file mode 100755 index 000000000..807a19b09 --- /dev/null +++ b/tests/instance-mysql @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +set -eu +script_name=$0 +die() { + echo >&2 "$@" + exit 1 +} + +about() { + die "usage: $script_name [backup | restore | configure | cleanup | setup] " +} + +#shellcheck disable=SC1007 +THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +cd "${THIS_DIR}" +#shellcheck disable=SC1090 +. ./.environment.sh + +# you have not removed set -u above, have you? + +[ -z "${DATA_DIR-}" ] && die "\$DATA_DIR must be defined." + +# --------------------------- + +[ $# -lt 1 ] && about + +./assert-crowdsec-not-running + +case "$1" in + backup) + [ $# -lt 2 ] && about + backup_file="$2" + # dirty fast cp. nothing should be accessing it right now, anyway. + fixtures/mysql backup $backup_file + ;; + restore) + [ $# -lt 2 ] && about + backup_file="$2" + [ -f "$backup_file" ] || die "missing file $backup_file" + fixtures/mysql restore $backup_file + ;; + configure) + fixtures/mysql configure + ;; + setup) + fixtures/mysql setup + ;; + cleanup) + fixtures/mysql cleanup + ;; + *) + about + ;; +esac; + diff --git a/tests/instance-sqlite b/tests/instance-sqlite index 6fa33d13f..aacba3f9e 100755 --- a/tests/instance-sqlite +++ b/tests/instance-sqlite @@ -28,10 +28,15 @@ cd "${THIS_DIR}" ./assert-crowdsec-not-running -DATA_DIR=$(yq <"${CONFIG_YAML}" '.config_paths.data_dir') DB_FILE="${DATA_DIR}/crowdsec.db" case "$1" in + configure) + ;; + setup) + ;; + cleanup) + ;; backup) [ $# -lt 2 ] && about backup_file="$2"