From 64c5fa7360ff84e9c4a5c72540eb7495b4734d77 Mon Sep 17 00:00:00 2001 From: "Thibault \"bui\" Koechlin" Date: Wed, 10 Jun 2020 12:14:27 +0200 Subject: [PATCH] CI: add a CI to test parsers (#67) --- .github/workflows/hub-ci.yml | 48 ++++++++++++++++++++++++++++++++++ README.md | 1 + cmd/crowdsec/acquisition.go | 34 ------------------------ cmd/crowdsec/main.go | 2 +- pkg/acquisition/file_reader.go | 28 ++++++++++++++++++++ pkg/parser/runtime.go | 25 +++++++++++++++--- pkg/types/utils.go | 16 ++++++++++++ 7 files changed, 115 insertions(+), 39 deletions(-) create mode 100644 .github/workflows/hub-ci.yml delete mode 100644 cmd/crowdsec/acquisition.go diff --git a/.github/workflows/hub-ci.yml b/.github/workflows/hub-ci.yml new file mode 100644 index 000000000..c7e549736 --- /dev/null +++ b/.github/workflows/hub-ci.yml @@ -0,0 +1,48 @@ +name: Hub-CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + name: Hub Parser/Scenario tests + runs-on: ubuntu-latest + steps: + - name: Set up Go 1.13 + uses: actions/setup-go@v1 + with: + go-version: 1.13 + id: go + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + - name: Build release + run: make release + - name: clone and build hub CI tool + run: | + git clone https://github.com/crowdsecurity/hub-tests.git + cd hub-tests + make + - name: Create crowdsec test env with all parsers from the release + run: | + cd crowdsec-pull + ./test_env.sh + cd tests + for i in `./cscli -c dev.yaml list parsers -a -o json | jq -r ".[].name" ` ; do + ./cscli -c dev.yaml install parser $i ; + done + - name: Setup hub ci in crowdsec + working-directory: ./crowdsec-pull/tests/ + run: | + cp -R ../../hub-tests/tests . + cp ../../hub-tests/main . + - name: Run the HUB CI + working-directory: ./crowdsec-pull/tests/ + run: | + for i in `find ./tests -mindepth 1 -maxdepth 1 -type d` ; do + echo "::group::Test-${i}" ; + ./main $i || (echo "::error file=${i}::Failed test for ${i}" ; diff ${i}"/results.yaml" ${i}"/results.yaml.fail") ; + echo "::endgroup::" ; + done ; diff --git a/README.md b/README.md index 34d8d09a4..8bb6a2a75 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ +

diff --git a/cmd/crowdsec/acquisition.go b/cmd/crowdsec/acquisition.go deleted file mode 100644 index 246cf3945..000000000 --- a/cmd/crowdsec/acquisition.go +++ /dev/null @@ -1,34 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/crowdsecurity/crowdsec/pkg/acquisition" -) - -func loadAcquisition() (*acquisition.FileAcquisCtx, error) { - var acquisitionCTX *acquisition.FileAcquisCtx - var err error - /*Init the acqusition : from cli or from acquis.yaml file*/ - if cConfig.SingleFile != "" { - var input acquisition.FileCtx - input.Filename = cConfig.SingleFile - input.Mode = acquisition.CATMODE - input.Labels = make(map[string]string) - input.Labels["type"] = cConfig.SingleFileLabel - acquisitionCTX, err = acquisition.InitReaderFromFileCtx([]acquisition.FileCtx{input}) - } else { /* Init file reader if we tail */ - acquisitionCTX, err = acquisition.InitReader(cConfig.AcquisitionFile) - } - if err != nil { - return nil, fmt.Errorf("unable to start file acquisition, bailout %v", err) - } - if acquisitionCTX == nil { - return nil, fmt.Errorf("no inputs to process") - } - if cConfig.Profiling { - acquisitionCTX.Profiling = true - } - - return acquisitionCTX, nil -} diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index cc9bbf85d..d3350db6d 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -294,7 +294,7 @@ func main() { log.Warningf("Starting processing data") //Init the acqusition : from cli or from acquis.yaml file - acquisitionCTX, err = loadAcquisition() + acquisitionCTX, err = acquisition.LoadAcquisitionConfig(cConfig) if err != nil { log.Fatalf("Failed to start acquisition : %s", err) } diff --git a/pkg/acquisition/file_reader.go b/pkg/acquisition/file_reader.go index 116ab2fab..ecb41711f 100644 --- a/pkg/acquisition/file_reader.go +++ b/pkg/acquisition/file_reader.go @@ -10,6 +10,7 @@ import ( "os" "strings" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket" "github.com/crowdsecurity/crowdsec/pkg/types" @@ -66,6 +67,33 @@ var ReaderHits = prometheus.NewCounterVec( []string{"source"}, ) +func LoadAcquisitionConfig(cConfig *csconfig.CrowdSec) (*FileAcquisCtx, error) { + var acquisitionCTX *FileAcquisCtx + var err error + /*Init the acqusition : from cli or from acquis.yaml file*/ + if cConfig.SingleFile != "" { + var input FileCtx + input.Filename = cConfig.SingleFile + input.Mode = CATMODE + input.Labels = make(map[string]string) + input.Labels["type"] = cConfig.SingleFileLabel + acquisitionCTX, err = InitReaderFromFileCtx([]FileCtx{input}) + } else { /* Init file reader if we tail */ + acquisitionCTX, err = InitReader(cConfig.AcquisitionFile) + } + if err != nil { + return nil, fmt.Errorf("unable to start file acquisition, bailout %v", err) + } + if acquisitionCTX == nil { + return nil, fmt.Errorf("no inputs to process") + } + if cConfig.Profiling { + acquisitionCTX.Profiling = true + } + + return acquisitionCTX, nil +} + func InitReader(cfg string) (*FileAcquisCtx, error) { var files []FileCtx diff --git a/pkg/parser/runtime.go b/pkg/parser/runtime.go index b255f21c0..385b23c93 100644 --- a/pkg/parser/runtime.go +++ b/pkg/parser/runtime.go @@ -227,6 +227,9 @@ func stageidx(stage string, stages []string) int { return -1 } +var ParseDump bool +var StageParseCache map[string]map[string]types.Event + func /*(u types.UnixParser)*/ Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error) { var event types.Event = xp @@ -250,7 +253,14 @@ func /*(u types.UnixParser)*/ Parse(ctx UnixParserCtx, xp types.Event, nodes []N log.Tracef("INPUT '%s'", event.Line.Raw) } + if ParseDump { + StageParseCache = make(map[string]map[string]types.Event) + } + for _, stage := range ctx.Stages { + if ParseDump { + StageParseCache[stage] = make(map[string]types.Event) + } /* if the node is forward in stages, seek to its stage */ /* this is for example used by testing system to inject logs in post-syslog-parsing phase*/ if stageidx(event.Stage, ctx.Stages) > stageidx(stage, ctx.Stages) { @@ -267,14 +277,14 @@ func /*(u types.UnixParser)*/ Parse(ctx UnixParserCtx, xp types.Event, nodes []N isStageOK := false for idx, node := range nodes { - clog := log.WithFields(log.Fields{ - "node-name": node.rn, - "stage": event.Stage, - }) //Only process current stage's nodes if event.Stage != node.Stage { continue } + clog := log.WithFields(log.Fields{ + "node-name": node.rn, + "stage": event.Stage, + }) clog.Tracef("Processing node %d/%d -> %s", idx, len(nodes), node.rn) if ctx.Profiling { node.Profiling = true @@ -286,6 +296,13 @@ func /*(u types.UnixParser)*/ Parse(ctx UnixParserCtx, xp types.Event, nodes []N clog.Tracef("node (%s) ret : %v", node.rn, ret) if ret { isStageOK = true + if ParseDump { + evtcopy := types.Event{} + if err := types.Clone(&event, &evtcopy); err != nil { + log.Fatalf("while cloning Event in parser : %s", err) + } + StageParseCache[stage][node.Name] = evtcopy + } } if ret && node.OnSuccess == "next_stage" { clog.Debugf("node successful, stop end stage %s", stage) diff --git a/pkg/types/utils.go b/pkg/types/utils.go index a5f8fded9..f7f119cfa 100644 --- a/pkg/types/utils.go +++ b/pkg/types/utils.go @@ -1,7 +1,9 @@ package types import ( + "bytes" "encoding/binary" + "encoding/gob" "fmt" "io" "net" @@ -93,3 +95,17 @@ func ConfigureLogger(clog *log.Logger) error { clog.SetLevel(logLevel) return nil } + +func Clone(a, b interface{}) error { + + buff := new(bytes.Buffer) + enc := gob.NewEncoder(buff) + dec := gob.NewDecoder(buff) + if err := enc.Encode(a); err != nil { + return fmt.Errorf("failed cloning %T", a) + } + if err := dec.Decode(b); err != nil { + return fmt.Errorf("failed cloning %T", b) + } + return nil +}