Mysql enabled tests (#1372)

* add msyql backend test

Co-authored-by: sabban <15465465+sabban@users.noreply.github.com>

.github/workflows/ci_bats_mysql.yaml

@@ -0,0 +1,58 @@
+name: BATS functional tests with MYSQL
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  build:
+    name: "Build the application"
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    services:
+      mariadb:
+        image: mysql:latest
+        env:
+          MYSQL_ROOT_PASSWORD: mysql
+        ports:
+          - 3306:3306
+    steps:
+
+    - name: "Set up Go 1.17"
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.17
+      id: go
+
+    - name: "Clone CrowdSec"
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+        submodules: true
+
+    - name: "Install bats dependencies"
+      run: |
+        sudo apt install daemonize netcat-openbsd
+        GO111MODULE=on go get github.com/mikefarah/yq/v4
+
+    - name: "BATS: build crowdsec"
+      run: make bats-clean bats-build
+      env:
+        DB_BACKEND: mysql
+        MYSQL_HOST: 127.0.0.1
+        MYSQL_PORT: 3306
+        MYSQL_PASSWORD: mysql
+        MYSQL_USER: root
+
+    - name: "BATS: run tests"
+      run: make bats-test
+      env:
+        DB_BACKEND: mysql
+        MYSQL_HOST: 127.0.0.1
+        MYSQL_PORT: 3306
+        MYSQL_PASSWORD: mysql
+        MYSQL_USER: root

tests/bats.mk

@@ -13,6 +13,7 @@ LOCAL_INIT_DIR = $(TEST_DIR)/local-init
 LOG_DIR = $(LOCAL_DIR)/var/log
 PID_DIR = $(LOCAL_DIR)/var/run
 PLUGIN_DIR = $(LOCAL_DIR)/lib/crowdsec/plugins
+DB_BACKEND := $(or $(DB_BACKEND), "sqlite")
 
 define ENV :=
 export TEST_DIR="$(TEST_DIR)"
@@ -24,6 +25,7 @@ export LOCAL_INIT_DIR="$(LOCAL_INIT_DIR)"
 export LOG_DIR="$(LOG_DIR)"
 export PID_DIR="$(PID_DIR)"
 export PLUGIN_DIR="$(PLUGIN_DIR)"
+export DB_BACKEND="$(DB_BACKEND)"
 endef
 
 bats-all: bats-clean bats-build bats-test

tests/fixtures/mysql

@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+
+set -eu
+script_name=$0
+
+die() {
+    echo >&2 "$@"
+    exit 1
+}
+
+MYSQL_HOST=${MYSQL_HOST:-127.0.0.1}
+MYSQL_PORT=${MYSQL_PORT:-3306}
+MYSQL_PASSWORD=${MYSQL_PASSWORD:-password}
+MYSQL_USER=${MYSQL_USER:-root}
+
+about() {
+    die "usage: $0 [ setup | cleanup | configure | backup | restore]"
+}
+
+check_mysql_client() {
+    if ! command -v mysql >/dev/null; then
+        die "missing required program 'mysql' as a mysql client (package mariadb-client-core-10.6 on debian like system)"
+    fi
+}
+
+send_mysql() {
+    if [ "$#" -lt 1 ]; then
+        die "missing requirement argument to send the mysql database"
+    fi
+
+    /bin/echo "$1" | mysql -h ${MYSQL_HOST} -u ${MYSQL_USER} --port=${MYSQL_PORT} -p${MYSQL_PASSWORD}
+}
+
+requirements() {
+    check_mysql_client
+}
+
+setup_database() {
+    send_mysql "CREATE DATABASE IF NOT EXISTS crowdsec;"
+    send_mysql "CREATE USER IF NOT EXISTS 'crowdsec' IDENTIFIED BY 'crowdsec';"
+    send_mysql "GRANT ALL PRIVILEGES ON crowdsec.* TO 'crowdsec';"
+}
+
+# cleanup is idempotent
+cleanup_database() {
+    send_mysql "DROP DATABASE IF EXISTS crowdsec;"
+    send_mysql "DROP USER IF EXISTS crowdsec;"
+}
+
+backup_database() {
+    if [ "$#" -lt 1 ]; then
+        die "missing file to backup database to"
+    fi
+    mysqldump -h${MYSQL_HOST} -u${MYSQL_USER} -p${MYSQL_PASSWORD} --databases crowdsec > $1
+}
+
+restore_database() {
+    if [ "$#" -lt 1 ]; then
+        die "missing file to restore database to"
+    fi
+    
+    if [ -f $1 ]; then
+        mysql -h ${MYSQL_HOST} -u ${MYSQL_USER} --port=${MYSQL_PORT} -p${MYSQL_PASSWORD} < $1
+    else
+        die "$2 doesn\'t exist"
+    fi
+    send_mysql "CREATE USER IF NOT EXISTS 'crowdsec' IDENTIFIED BY 'crowdsec';"
+    send_mysql "GRANT ALL PRIVILEGES ON crowdsec.* TO 'crowdsec';"
+}
+
+setup_configuration() {
+    MYSQL_PORT=${MYSQL_PORT} MYSQL_HOST=${MYSQL_HOST} yq '
+        .db_config.type="mysql"|
+        .db_config.user="crowdsec" |
+        .db_config.password="crowdsec" |
+        .db_config.db_name="crowdsec"  |
+        .db_config.host=strenv(MYSQL_HOST) |
+        .db_config.port=env(MYSQL_PORT) |
+        del(.db_config.db_path) 
+    ' -i "${CONFIG_YAML}" 
+}
+
+
+case "$1" in
+    setup)
+        setup_database
+        ;;
+    cleanup)
+        cleanup_database
+        ;;
+    configure)
+        setup_configuration
+        ;;
+    backup)
+        if [ "$#" -lt 2 ]; then
+            die "missing file to restore backup to"
+        fi
+        backup_database $2
+        ;;
+    restore)
+        if [ "$#" -lt 2 ]; then
+            die "missing file to restore restore to"
+        fi
+        restore_database $2
+        ;;
+    *)
+        about
+        ;;
+esac;

tests/instance-data

@@ -25,6 +25,7 @@ cd "${THIS_DIR}"
 [ -z "${CSCLI-}" ] && die "\$CSCLI must be defined."
 [ -z "${LOCAL_INIT_DIR-}" ] && die "\$LOCAL_INIT_DIR must be defined."
 [ -z "${PLUGIN_DIR-}" ] && die "\$PLUGIN_DIR must be defined."
+[ -z "${DB_BACKEND-}" ] && die "\$DB_BACKEND must be defined."
 
 if [ ! -f "${CSCLI}" ]; then
     die "${CSCLI} is missing. Please build (with 'make bats-build') or install it."
@@ -40,6 +41,8 @@ export CONFIG_DIR
 
 remove_init_data() {
     rm -rf -- "${LOCAL_DIR:?}/${REL_CONFIG_DIR}"/* "${LOCAL_DIR:?}/${REL_DATA_DIR:?}"/*
+
+    ./instance-db cleanup
 }
 
 config_generate() {
@@ -82,6 +85,8 @@ make_init_data() {
     mkdir -p "${CONFIG_DIR}/notifications"
 
     config_generate
+    ./instance-db configure
+    ./instance-db setup
 
     mkdir -p "${CONFIG_DIR}/hub"
     "${CSCLI}" machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto

tests/instance-db

@@ -4,4 +4,15 @@
 THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
 cd "${THIS_DIR}"
 
-./instance-sqlite "$@"
+case "$DB_BACKEND" in
+    sqlite)
+        ./instance-sqlite "$@"
+        ;;
+    mysql)
+        ./instance-mysql "$@"
+        ;;
+    *)
+        echo >&2 "unknown $DB_BACKEND"
+        exit 1
+        ;;
+esac;

tests/instance-mysql

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+set -eu
+script_name=$0
+die() {
+      echo >&2 "$@"
+      exit 1
+}
+
+about() {
+    die "usage: $script_name [backup | restore | configure | cleanup | setup] <backup_file>"
+}
+
+#shellcheck disable=SC1007
+THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+cd "${THIS_DIR}"
+#shellcheck disable=SC1090
+. ./.environment.sh
+
+# you have not removed set -u above, have you?
+
+[ -z "${DATA_DIR-}" ] && die "\$DATA_DIR must be defined."
+
+# ---------------------------
+
+[ $# -lt 1 ] && about
+
+./assert-crowdsec-not-running
+
+case "$1" in
+     backup)
+         [ $# -lt 2 ] && about
+         backup_file="$2"
+         # dirty fast cp. nothing should be accessing it right now, anyway.
+         fixtures/mysql backup $backup_file
+         ;;
+     restore)
+         [ $# -lt 2 ] && about
+         backup_file="$2"
+         [ -f "$backup_file" ] || die "missing file $backup_file"
+         fixtures/mysql restore $backup_file
+         ;;
+     configure)
+         fixtures/mysql configure
+         ;;
+     setup)
+         fixtures/mysql setup
+         ;;
+     cleanup)
+         fixtures/mysql cleanup
+         ;;
+      *)
+         about
+         ;;
+esac;
+

tests/instance-sqlite

@@ -28,10 +28,15 @@ cd "${THIS_DIR}"
 
 ./assert-crowdsec-not-running
 
-DATA_DIR=$(yq <"${CONFIG_YAML}" '.config_paths.data_dir')
 DB_FILE="${DATA_DIR}/crowdsec.db"
 
 case "$1" in
+    configure)
+        ;;
+    setup)
+        ;;
+    cleanup)
+        ;;
     backup)
         [ $# -lt 2 ] && about
         backup_file="$2"