Parallel hubtests (#2667)

* generate hub tests in python
* run hub tests in 3 batches at the same time (hardcoded)
This commit is contained in:
mmetc 2023-12-15 18:30:20 +01:00 committed by GitHub
parent a79fcaf378
commit c2c173ac7e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 74 additions and 44 deletions

View file

@ -15,7 +15,7 @@ jobs:
build: build:
strategy: strategy:
matrix: matrix:
go-version: ["1.21.5"] test-file: ["hub-1.bats", "hub-2.bats", "hub-3.bats"]
name: "Build + tests" name: "Build + tests"
runs-on: ubuntu-latest runs-on: ubuntu-latest
@ -33,10 +33,10 @@ jobs:
fetch-depth: 0 fetch-depth: 0
submodules: true submodules: true
- name: "Set up Go ${{ matrix.go-version }}" - name: "Set up Go"
uses: actions/setup-go@v4 uses: actions/setup-go@v4
with: with:
go-version: ${{ matrix.go-version }} go-version: "1.21.5"
- name: "Install bats dependencies" - name: "Install bats dependencies"
env: env:
@ -48,7 +48,9 @@ jobs:
run: make bats-clean bats-build bats-fixture BUILD_STATIC=1 run: make bats-clean bats-build bats-fixture BUILD_STATIC=1
- name: "Run hub tests" - name: "Run hub tests"
run: make bats-test-hub run: |
./test/bin/generate-hub-tests
./test/run-tests test/dyn-bats/${{ matrix.test-file }}
- name: "Collect hub coverage" - name: "Collect hub coverage"
run: ./test/bin/collect-hub-coverage >> $GITHUB_ENV run: ./test/bin/collect-hub-coverage >> $GITHUB_ENV

View file

@ -32,10 +32,6 @@ env:
jobs: jobs:
build: build:
strategy:
matrix:
go-version: ["1.21.5"]
name: "Build + tests" name: "Build + tests"
runs-on: ubuntu-latest runs-on: ubuntu-latest
services: services:
@ -128,10 +124,10 @@ jobs:
fetch-depth: 0 fetch-depth: 0
submodules: false submodules: false
- name: "Set up Go ${{ matrix.go-version }}" - name: "Set up Go"
uses: actions/setup-go@v4 uses: actions/setup-go@v4
with: with:
go-version: ${{ matrix.go-version }} go-version: "1.21.5"
- name: Build and run tests, static - name: Build and run tests, static
run: | run: |

View file

@ -7,44 +7,13 @@ THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
# shellcheck disable=SC1091 # shellcheck disable=SC1091
. "${THIS_DIR}/../.environment.sh" . "${THIS_DIR}/../.environment.sh"
cscli() {
"${CSCLI}" "$@"
}
"${TEST_DIR}/instance-data" load "${TEST_DIR}/instance-data" load
hubdir="${LOCAL_DIR}/hub-tests" hubdir="${LOCAL_DIR}/hub-tests"
git clone --depth 1 https://github.com/crowdsecurity/hub.git "${hubdir}" >/dev/null 2>&1 || (cd "${hubdir}"; git pull) git clone --depth 1 https://github.com/crowdsecurity/hub.git "${hubdir}" >/dev/null 2>&1 || (cd "${hubdir}"; git pull)
HUBTESTS_BATS="${TEST_DIR}/dyn-bats/hub.bats"
cat << EOT > "${HUBTESTS_BATS}"
set -u
setup_file() {
load "../lib/setup_file.sh"
}
teardown_file() {
load "../lib/teardown_file.sh"
}
setup() {
load "../lib/setup.sh"
}
EOT
echo "Generating hub tests..." echo "Generating hub tests..."
for testname in $("${CSCLI}" --crowdsec "${CROWDSEC}" --cscli "${CSCLI}" hubtest --hub "${hubdir}" list -o json | jq -r '.[] | .Name'); do python3 "$THIS_DIR/generate-hub-tests.py" \
cat << EOT >> "${HUBTESTS_BATS}" <("${CSCLI}" --crowdsec "${CROWDSEC}" --cscli "${CSCLI}" hubtest --hub "${hubdir}" list -o json) \
"${TEST_DIR}/dyn-bats/"
@test "${testname}" {
run "\${CSCLI}" --crowdsec "\${CROWDSEC}" --cscli "\${CSCLI}" --hub "${hubdir}" hubtest run "${testname}" --clean
# in case of error, need to see what went wrong
echo "\$output"
assert_success
}
EOT
done

View file

@ -0,0 +1,63 @@
#!/usr/bin/env python3
import json
import pathlib
import os
import sys
import textwrap
test_header = """
set -u
setup_file() {
load "../lib/setup_file.sh"
}
teardown_file() {
load "../lib/teardown_file.sh"
}
setup() {
load "../lib/setup.sh"
}
"""
def write_chunk(target_dir, n, chunk):
with open(target_dir / f"hub-{n}.bats", "w") as f:
f.write(test_header)
for test in chunk:
cscli = os.environ['CSCLI']
crowdsec = os.environ['CROWDSEC']
testname = test['Name']
hubdir = os.environ['LOCAL_DIR'] + '/hub-tests'
f.write(textwrap.dedent(f"""
@test "{testname}" {{
run "{cscli}" \\
--crowdsec "{crowdsec}" \\
--cscli "{cscli}" \\
--hub "{hubdir}" \\
hubtest run "{testname}" \\
--clean
echo "$output"
assert_success
}}
"""))
def main():
hubtests_json = sys.argv[1]
target_dir = sys.argv[2]
with open(hubtests_json) as f:
j = json.load(f)
chunk_size = len(j) // 3 + 1
n = 1
for i in range(0, len(j), chunk_size):
chunk = j[i:i + chunk_size]
write_chunk(pathlib.Path(target_dir), n, chunk)
n += 1
if __name__ == "__main__":
main()