Parallel hubtests (#2667)
* generate hub tests in python * run hub tests in 3 batches at the same time (hardcoded)
This commit is contained in:
parent
a79fcaf378
commit
c2c173ac7e
10
.github/workflows/bats-hub.yml
vendored
10
.github/workflows/bats-hub.yml
vendored
|
@ -15,7 +15,7 @@ jobs:
|
||||||
build:
|
build:
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
go-version: ["1.21.5"]
|
test-file: ["hub-1.bats", "hub-2.bats", "hub-3.bats"]
|
||||||
|
|
||||||
name: "Build + tests"
|
name: "Build + tests"
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
@ -33,10 +33,10 @@ jobs:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
submodules: true
|
submodules: true
|
||||||
|
|
||||||
- name: "Set up Go ${{ matrix.go-version }}"
|
- name: "Set up Go"
|
||||||
uses: actions/setup-go@v4
|
uses: actions/setup-go@v4
|
||||||
with:
|
with:
|
||||||
go-version: ${{ matrix.go-version }}
|
go-version: "1.21.5"
|
||||||
|
|
||||||
- name: "Install bats dependencies"
|
- name: "Install bats dependencies"
|
||||||
env:
|
env:
|
||||||
|
@ -48,7 +48,9 @@ jobs:
|
||||||
run: make bats-clean bats-build bats-fixture BUILD_STATIC=1
|
run: make bats-clean bats-build bats-fixture BUILD_STATIC=1
|
||||||
|
|
||||||
- name: "Run hub tests"
|
- name: "Run hub tests"
|
||||||
run: make bats-test-hub
|
run: |
|
||||||
|
./test/bin/generate-hub-tests
|
||||||
|
./test/run-tests test/dyn-bats/${{ matrix.test-file }}
|
||||||
|
|
||||||
- name: "Collect hub coverage"
|
- name: "Collect hub coverage"
|
||||||
run: ./test/bin/collect-hub-coverage >> $GITHUB_ENV
|
run: ./test/bin/collect-hub-coverage >> $GITHUB_ENV
|
||||||
|
|
8
.github/workflows/go-tests.yml
vendored
8
.github/workflows/go-tests.yml
vendored
|
@ -32,10 +32,6 @@ env:
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build:
|
build:
|
||||||
strategy:
|
|
||||||
matrix:
|
|
||||||
go-version: ["1.21.5"]
|
|
||||||
|
|
||||||
name: "Build + tests"
|
name: "Build + tests"
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
services:
|
services:
|
||||||
|
@ -128,10 +124,10 @@ jobs:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
submodules: false
|
submodules: false
|
||||||
|
|
||||||
- name: "Set up Go ${{ matrix.go-version }}"
|
- name: "Set up Go"
|
||||||
uses: actions/setup-go@v4
|
uses: actions/setup-go@v4
|
||||||
with:
|
with:
|
||||||
go-version: ${{ matrix.go-version }}
|
go-version: "1.21.5"
|
||||||
|
|
||||||
- name: Build and run tests, static
|
- name: Build and run tests, static
|
||||||
run: |
|
run: |
|
||||||
|
|
|
@ -7,44 +7,13 @@ THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
. "${THIS_DIR}/../.environment.sh"
|
. "${THIS_DIR}/../.environment.sh"
|
||||||
|
|
||||||
cscli() {
|
|
||||||
"${CSCLI}" "$@"
|
|
||||||
}
|
|
||||||
|
|
||||||
"${TEST_DIR}/instance-data" load
|
"${TEST_DIR}/instance-data" load
|
||||||
|
|
||||||
hubdir="${LOCAL_DIR}/hub-tests"
|
hubdir="${LOCAL_DIR}/hub-tests"
|
||||||
git clone --depth 1 https://github.com/crowdsecurity/hub.git "${hubdir}" >/dev/null 2>&1 || (cd "${hubdir}"; git pull)
|
git clone --depth 1 https://github.com/crowdsecurity/hub.git "${hubdir}" >/dev/null 2>&1 || (cd "${hubdir}"; git pull)
|
||||||
|
|
||||||
HUBTESTS_BATS="${TEST_DIR}/dyn-bats/hub.bats"
|
|
||||||
|
|
||||||
cat << EOT > "${HUBTESTS_BATS}"
|
|
||||||
set -u
|
|
||||||
|
|
||||||
setup_file() {
|
|
||||||
load "../lib/setup_file.sh"
|
|
||||||
}
|
|
||||||
|
|
||||||
teardown_file() {
|
|
||||||
load "../lib/teardown_file.sh"
|
|
||||||
}
|
|
||||||
|
|
||||||
setup() {
|
|
||||||
load "../lib/setup.sh"
|
|
||||||
}
|
|
||||||
|
|
||||||
EOT
|
|
||||||
|
|
||||||
echo "Generating hub tests..."
|
echo "Generating hub tests..."
|
||||||
|
|
||||||
for testname in $("${CSCLI}" --crowdsec "${CROWDSEC}" --cscli "${CSCLI}" hubtest --hub "${hubdir}" list -o json | jq -r '.[] | .Name'); do
|
python3 "$THIS_DIR/generate-hub-tests.py" \
|
||||||
cat << EOT >> "${HUBTESTS_BATS}"
|
<("${CSCLI}" --crowdsec "${CROWDSEC}" --cscli "${CSCLI}" hubtest --hub "${hubdir}" list -o json) \
|
||||||
|
"${TEST_DIR}/dyn-bats/"
|
||||||
@test "${testname}" {
|
|
||||||
run "\${CSCLI}" --crowdsec "\${CROWDSEC}" --cscli "\${CSCLI}" --hub "${hubdir}" hubtest run "${testname}" --clean
|
|
||||||
# in case of error, need to see what went wrong
|
|
||||||
echo "\$output"
|
|
||||||
assert_success
|
|
||||||
}
|
|
||||||
EOT
|
|
||||||
done
|
|
||||||
|
|
63
test/bin/generate-hub-tests.py
Normal file
63
test/bin/generate-hub-tests.py
Normal file
|
@ -0,0 +1,63 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import json
|
||||||
|
import pathlib
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import textwrap
|
||||||
|
|
||||||
|
test_header = """
|
||||||
|
set -u
|
||||||
|
|
||||||
|
setup_file() {
|
||||||
|
load "../lib/setup_file.sh"
|
||||||
|
}
|
||||||
|
|
||||||
|
teardown_file() {
|
||||||
|
load "../lib/teardown_file.sh"
|
||||||
|
}
|
||||||
|
|
||||||
|
setup() {
|
||||||
|
load "../lib/setup.sh"
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def write_chunk(target_dir, n, chunk):
|
||||||
|
with open(target_dir / f"hub-{n}.bats", "w") as f:
|
||||||
|
f.write(test_header)
|
||||||
|
for test in chunk:
|
||||||
|
cscli = os.environ['CSCLI']
|
||||||
|
crowdsec = os.environ['CROWDSEC']
|
||||||
|
testname = test['Name']
|
||||||
|
hubdir = os.environ['LOCAL_DIR'] + '/hub-tests'
|
||||||
|
f.write(textwrap.dedent(f"""
|
||||||
|
@test "{testname}" {{
|
||||||
|
run "{cscli}" \\
|
||||||
|
--crowdsec "{crowdsec}" \\
|
||||||
|
--cscli "{cscli}" \\
|
||||||
|
--hub "{hubdir}" \\
|
||||||
|
hubtest run "{testname}" \\
|
||||||
|
--clean
|
||||||
|
echo "$output"
|
||||||
|
assert_success
|
||||||
|
}}
|
||||||
|
"""))
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
hubtests_json = sys.argv[1]
|
||||||
|
target_dir = sys.argv[2]
|
||||||
|
|
||||||
|
with open(hubtests_json) as f:
|
||||||
|
j = json.load(f)
|
||||||
|
chunk_size = len(j) // 3 + 1
|
||||||
|
n = 1
|
||||||
|
for i in range(0, len(j), chunk_size):
|
||||||
|
chunk = j[i:i + chunk_size]
|
||||||
|
write_chunk(pathlib.Path(target_dir), n, chunk)
|
||||||
|
n += 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
Loading…
Reference in a new issue