From 11b18430704471dff0ca754a65647b5f2ba6e5e4 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Thu, 7 Oct 2021 18:21:08 +0200 Subject: [PATCH] update --- cmd/crowdsec-cli/decisions.go | 4 +-- pkg/apiserver/apic.go | 2 +- pkg/apiserver/controllers/v1/alerts.go | 9 +++-- pkg/database/alerts.go | 48 +++++++++++++++----------- 4 files changed, 36 insertions(+), 27 deletions(-) diff --git a/cmd/crowdsec-cli/decisions.go b/cmd/crowdsec-cli/decisions.go index 59c5aaa45..34eeb7c98 100644 --- a/cmd/crowdsec-cli/decisions.go +++ b/cmd/crowdsec-cli/decisions.go @@ -25,9 +25,10 @@ var Client *apiclient.ApiClient func DecisionsToTable(alerts *models.GetAlertsResponse) error { /*here we cheat a bit : to make it more readable for the user, we dedup some entries*/ var spamLimit map[string]bool = make(map[string]bool) + /*process in reverse order to keep the latest item only*/ /*process in reverse order to keep the latest item only*/ - for aIdx := len(*alerts) - 1; aIdx >= 0; aIdx-- { + for aIdx := 0; aIdx < len(*alerts); aIdx++ { alertItem := (*alerts)[aIdx] newDecisions := make([]*models.Decision, 0) for _, decisionItem := range alertItem.Decisions { @@ -227,7 +228,6 @@ cscli decisions list -t ban if err != nil { log.Fatalf("Unable to list decisions : %v", err.Error()) } - err = DecisionsToTable(alerts) if err != nil { log.Fatalf("unable to list decisions : %v", err.Error()) diff --git a/pkg/apiserver/apic.go b/pkg/apiserver/apic.go index 9fa8ea7eb..7a069caf2 100644 --- a/pkg/apiserver/apic.go +++ b/pkg/apiserver/apic.go @@ -91,7 +91,7 @@ func AlertToSignal(alert *models.Alert, scenarioTrust string, keepDecisions bool ScenarioTrust: &scenarioTrust, } if keepDecisions { - log.Printf("Keeping decisions to send to CAPI") + log.Debugf("Keeping decisions to send to CAPI") signal.Decisions = alert.Decisions } return signal diff --git a/pkg/apiserver/controllers/v1/alerts.go b/pkg/apiserver/controllers/v1/alerts.go index a103952be..37df23337 100644 --- a/pkg/apiserver/controllers/v1/alerts.go +++ b/pkg/apiserver/controllers/v1/alerts.go @@ -150,22 +150,25 @@ func (c *Controller) CreateAlert(gctx *gin.Context) { } } - alerts, err := c.DBClient.CreateAlert(machineID, input) + alertsID, alertsToSend, err := c.DBClient.CreateAlert(machineID, input) if err != nil { c.HandleDBErrors(gctx, err) return } + for _, alert := range alertsToSend { + alert.MachineID = machineID + } if c.CAPIChan != nil { select { - case c.CAPIChan <- input: + case c.CAPIChan <- alertsToSend: log.Debug("alert sent to CAPI channel") default: log.Warning("Cannot send alert to Central API channel") } } - gctx.JSON(http.StatusCreated, alerts) + gctx.JSON(http.StatusCreated, alertsID) return } diff --git a/pkg/database/alerts.go b/pkg/database/alerts.go index 2a8352a2a..202083d44 100644 --- a/pkg/database/alerts.go +++ b/pkg/database/alerts.go @@ -90,28 +90,31 @@ func formatAlertAsString(machineId string, alert *models.Alert) []string { return retStr } -func (c *Client) CreateAlert(machineID string, alertList []*models.Alert) ([]string, error) { +func (c *Client) CreateAlert(machineID string, alertList []*models.Alert) ([]string, []*models.Alert, error) { pageStart := 0 pageEnd := bulkSize ret := []string{} + alertListRet := make([]*models.Alert, 0) for { if pageEnd >= len(alertList) { - results, err := c.CreateAlertBulk(machineID, alertList[pageStart:]) + results, alerts, err := c.CreateAlertBulk(machineID, alertList[pageStart:]) if err != nil { - return []string{}, fmt.Errorf("unable to create alerts: %s", err) + return []string{}, alertListRet, fmt.Errorf("unable to create alerts: %s", err) } ret = append(ret, results...) + alertListRet = append(alertListRet, alerts...) break } - results, err := c.CreateAlertBulk(machineID, alertList[pageStart:pageEnd]) + results, alerts, err := c.CreateAlertBulk(machineID, alertList[pageStart:pageEnd]) if err != nil { - return []string{}, fmt.Errorf("unable to create alerts: %s", err) + return []string{}, alertListRet, fmt.Errorf("unable to create alerts: %s", err) } ret = append(ret, results...) + alertListRet = append(alertListRet, alerts...) pageStart += bulkSize pageEnd += bulkSize } - return ret, nil + return ret, alertListRet, nil } /*We can't bulk both the alert and the decision at the same time. With new consensus, we want to bulk a single alert with a lot of decisions.*/ @@ -278,7 +281,7 @@ func (c *Client) UpdateCommunityBlocklist(alertItem *models.Alert) (int, int, in return alertRef.ID, inserted, deleted, nil } -func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([]string, error) { +func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([]string, []*models.Alert, error) { ret := []string{} bulkSize := 20 @@ -293,19 +296,19 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ owner, err := c.QueryMachineByID(machineId) if err != nil { if errors.Cause(err) != UserNotExists { - return []string{}, errors.Wrapf(QueryFail, "machine '%s': %s", alertItem.MachineID, err) + return []string{}, alertList, errors.Wrapf(QueryFail, "machine '%s': %s", alertItem.MachineID, err) } c.Log.Debugf("CreateAlertBulk: Machine Id %s doesn't exist", machineId) owner = nil } startAtTime, err := time.Parse(time.RFC3339, *alertItem.StartAt) if err != nil { - return []string{}, errors.Wrapf(ParseTimeFail, "start_at field time '%s': %s", *alertItem.StartAt, err) + return []string{}, alertList, errors.Wrapf(ParseTimeFail, "start_at field time '%s': %s", *alertItem.StartAt, err) } stopAtTime, err := time.Parse(time.RFC3339, *alertItem.StopAt) if err != nil { - return []string{}, errors.Wrapf(ParseTimeFail, "stop_at field time '%s': %s", *alertItem.StopAt, err) + return []string{}, alertList, errors.Wrapf(ParseTimeFail, "stop_at field time '%s': %s", *alertItem.StopAt, err) } /*display proper alert in logs*/ for _, disp := range formatAlertAsString(machineId, alertItem) { @@ -321,11 +324,11 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ for i, eventItem := range alertItem.Events { ts, err := time.Parse(time.RFC3339, *eventItem.Timestamp) if err != nil { - return []string{}, errors.Wrapf(ParseTimeFail, "event timestamp '%s' : %s", *eventItem.Timestamp, err) + return []string{}, alertList, errors.Wrapf(ParseTimeFail, "event timestamp '%s' : %s", *eventItem.Timestamp, err) } marshallMetas, err := json.Marshal(eventItem.Meta) if err != nil { - return []string{}, errors.Wrapf(MarshalFail, "event meta '%v' : %s", eventItem.Meta, err) + return []string{}, alertList, errors.Wrapf(MarshalFail, "event meta '%v' : %s", eventItem.Meta, err) } //the serialized field is too big, let's try to progressively strip it @@ -343,7 +346,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ marshallMetas, err = json.Marshal(eventItem.Meta) if err != nil { - return []string{}, errors.Wrapf(MarshalFail, "event meta '%v' : %s", eventItem.Meta, err) + return []string{}, alertList, errors.Wrapf(MarshalFail, "event meta '%v' : %s", eventItem.Meta, err) } if event.SerializedValidator(string(marshallMetas)) == nil { valid = true @@ -372,7 +375,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ } events, err = c.Ent.Event.CreateBulk(eventBulk...).Save(c.CTX) if err != nil { - return []string{}, errors.Wrapf(BulkError, "creating alert events: %s", err) + return []string{}, alertList, errors.Wrapf(BulkError, "creating alert events: %s", err) } } @@ -385,7 +388,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ } metas, err = c.Ent.Meta.CreateBulk(metaBulk...).Save(c.CTX) if err != nil { - return []string{}, errors.Wrapf(BulkError, "creating alert meta: %s", err) + return []string{}, alertList, errors.Wrapf(BulkError, "creating alert meta: %s", err) } } @@ -402,14 +405,14 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ duration, err := time.ParseDuration(*decisionItem.Duration) if err != nil { - return []string{}, errors.Wrapf(ParseDurationFail, "decision duration '%v' : %s", decisionItem.Duration, err) + return []string{}, alertList, errors.Wrapf(ParseDurationFail, "decision duration '%v' : %s", decisionItem.Duration, err) } /*if the scope is IP or Range, convert the value to integers */ if strings.ToLower(*decisionItem.Scope) == "ip" || strings.ToLower(*decisionItem.Scope) == "range" { sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(*decisionItem.Value) if err != nil { - return []string{}, errors.Wrapf(ParseDurationFail, "invalid addr/range %s : %s", *decisionItem.Value, err) + return []string{}, alertList, errors.Wrapf(ParseDurationFail, "invalid addr/range %s : %s", *decisionItem.Value, err) } } decisionBulk[i] = c.Ent.Decision.Create(). @@ -428,9 +431,12 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ } decisions, err = c.Ent.Decision.CreateBulk(decisionBulk...).Save(c.CTX) if err != nil { - return []string{}, errors.Wrapf(BulkError, "creating alert decisions: %s", err) + return []string{}, alertList, errors.Wrapf(BulkError, "creating alert decisions: %s", err) } + for i, decisionItem := range alertItem.Decisions { + decisionItem.ID = int64(decisions[i].ID) + } } alertB := c.Ent.Alert. @@ -466,7 +472,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ if len(bulk) == bulkSize { alerts, err := c.Ent.Alert.CreateBulk(bulk...).Save(c.CTX) if err != nil { - return []string{}, errors.Wrapf(BulkError, "bulk creating alert : %s", err) + return []string{}, alertList, errors.Wrapf(BulkError, "bulk creating alert : %s", err) } for _, alert := range alerts { ret = append(ret, strconv.Itoa(alert.ID)) @@ -482,14 +488,14 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ alerts, err := c.Ent.Alert.CreateBulk(bulk...).Save(c.CTX) if err != nil { - return []string{}, errors.Wrapf(BulkError, "leftovers creating alert : %s", err) + return []string{}, alertList, errors.Wrapf(BulkError, "leftovers creating alert : %s", err) } for _, alert := range alerts { ret = append(ret, strconv.Itoa(alert.ID)) } - return ret, nil + return ret, alertList, nil } func BuildAlertRequestFromFilter(alerts *ent.AlertQuery, filter map[string][]string) (*ent.AlertQuery, error) {