From c2c173ac7e7eb6689ce076e40c624264a9ad73a3 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Fri, 15 Dec 2023 18:30:20 +0100 Subject: [PATCH] Parallel hubtests (#2667) * generate hub tests in python * run hub tests in 3 batches at the same time (hardcoded) --- .github/workflows/bats-hub.yml | 10 +++--- .github/workflows/go-tests.yml | 8 ++--- test/bin/generate-hub-tests | 37 ++------------------ test/bin/generate-hub-tests.py | 63 ++++++++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 44 deletions(-) create mode 100644 test/bin/generate-hub-tests.py diff --git a/.github/workflows/bats-hub.yml b/.github/workflows/bats-hub.yml index b9326b462..5520b0dac 100644 --- a/.github/workflows/bats-hub.yml +++ b/.github/workflows/bats-hub.yml @@ -15,7 +15,7 @@ jobs: build: strategy: matrix: - go-version: ["1.21.5"] + test-file: ["hub-1.bats", "hub-2.bats", "hub-3.bats"] name: "Build + tests" runs-on: ubuntu-latest @@ -33,10 +33,10 @@ jobs: fetch-depth: 0 submodules: true - - name: "Set up Go ${{ matrix.go-version }}" + - name: "Set up Go" uses: actions/setup-go@v4 with: - go-version: ${{ matrix.go-version }} + go-version: "1.21.5" - name: "Install bats dependencies" env: @@ -48,7 +48,9 @@ jobs: run: make bats-clean bats-build bats-fixture BUILD_STATIC=1 - name: "Run hub tests" - run: make bats-test-hub + run: | + ./test/bin/generate-hub-tests + ./test/run-tests test/dyn-bats/${{ matrix.test-file }} - name: "Collect hub coverage" run: ./test/bin/collect-hub-coverage >> $GITHUB_ENV diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 78bed2a40..4cc118e66 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -32,10 +32,6 @@ env: jobs: build: - strategy: - matrix: - go-version: ["1.21.5"] - name: "Build + tests" runs-on: ubuntu-latest services: @@ -128,10 +124,10 @@ jobs: fetch-depth: 0 submodules: false - - name: "Set up Go ${{ matrix.go-version }}" + - name: "Set up Go" uses: actions/setup-go@v4 with: - go-version: ${{ matrix.go-version }} + go-version: "1.21.5" - name: Build and run tests, static run: | diff --git a/test/bin/generate-hub-tests b/test/bin/generate-hub-tests index 21031285c..658cc33a7 100755 --- a/test/bin/generate-hub-tests +++ b/test/bin/generate-hub-tests @@ -7,44 +7,13 @@ THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) # shellcheck disable=SC1091 . "${THIS_DIR}/../.environment.sh" -cscli() { - "${CSCLI}" "$@" -} - "${TEST_DIR}/instance-data" load hubdir="${LOCAL_DIR}/hub-tests" git clone --depth 1 https://github.com/crowdsecurity/hub.git "${hubdir}" >/dev/null 2>&1 || (cd "${hubdir}"; git pull) -HUBTESTS_BATS="${TEST_DIR}/dyn-bats/hub.bats" - -cat << EOT > "${HUBTESTS_BATS}" -set -u - -setup_file() { - load "../lib/setup_file.sh" -} - -teardown_file() { - load "../lib/teardown_file.sh" -} - -setup() { - load "../lib/setup.sh" -} - -EOT - echo "Generating hub tests..." -for testname in $("${CSCLI}" --crowdsec "${CROWDSEC}" --cscli "${CSCLI}" hubtest --hub "${hubdir}" list -o json | jq -r '.[] | .Name'); do - cat << EOT >> "${HUBTESTS_BATS}" - -@test "${testname}" { - run "\${CSCLI}" --crowdsec "\${CROWDSEC}" --cscli "\${CSCLI}" --hub "${hubdir}" hubtest run "${testname}" --clean - # in case of error, need to see what went wrong - echo "\$output" - assert_success -} -EOT -done +python3 "$THIS_DIR/generate-hub-tests.py" \ + <("${CSCLI}" --crowdsec "${CROWDSEC}" --cscli "${CSCLI}" hubtest --hub "${hubdir}" list -o json) \ + "${TEST_DIR}/dyn-bats/" diff --git a/test/bin/generate-hub-tests.py b/test/bin/generate-hub-tests.py new file mode 100644 index 000000000..48f296776 --- /dev/null +++ b/test/bin/generate-hub-tests.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +import json +import pathlib +import os +import sys +import textwrap + +test_header = """ +set -u + +setup_file() { + load "../lib/setup_file.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" +} +""" + + +def write_chunk(target_dir, n, chunk): + with open(target_dir / f"hub-{n}.bats", "w") as f: + f.write(test_header) + for test in chunk: + cscli = os.environ['CSCLI'] + crowdsec = os.environ['CROWDSEC'] + testname = test['Name'] + hubdir = os.environ['LOCAL_DIR'] + '/hub-tests' + f.write(textwrap.dedent(f""" + @test "{testname}" {{ + run "{cscli}" \\ + --crowdsec "{crowdsec}" \\ + --cscli "{cscli}" \\ + --hub "{hubdir}" \\ + hubtest run "{testname}" \\ + --clean + echo "$output" + assert_success + }} + """)) + + +def main(): + hubtests_json = sys.argv[1] + target_dir = sys.argv[2] + + with open(hubtests_json) as f: + j = json.load(f) + chunk_size = len(j) // 3 + 1 + n = 1 + for i in range(0, len(j), chunk_size): + chunk = j[i:i + chunk_size] + write_chunk(pathlib.Path(target_dir), n, chunk) + n += 1 + + +if __name__ == "__main__": + main()