281 lines
6.9 KiB
Go
281 lines
6.9 KiB
Go
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/jszwec/csvutil"
|
|
log "github.com/sirupsen/logrus"
|
|
"github.com/spf13/cobra"
|
|
|
|
"github.com/crowdsecurity/go-cs-lib/ptr"
|
|
"github.com/crowdsecurity/go-cs-lib/slicetools"
|
|
|
|
"github.com/crowdsecurity/crowdsec/pkg/models"
|
|
"github.com/crowdsecurity/crowdsec/pkg/types"
|
|
)
|
|
|
|
// decisionRaw is only used to unmarshall json/csv decisions
|
|
type decisionRaw struct {
|
|
Duration string `csv:"duration,omitempty" json:"duration,omitempty"`
|
|
Scenario string `csv:"reason,omitempty" json:"reason,omitempty"`
|
|
Scope string `csv:"scope,omitempty" json:"scope,omitempty"`
|
|
Type string `csv:"type,omitempty" json:"type,omitempty"`
|
|
Value string `csv:"value" json:"value"`
|
|
}
|
|
|
|
func parseDecisionList(content []byte, format string) ([]decisionRaw, error) {
|
|
ret := []decisionRaw{}
|
|
|
|
switch format {
|
|
case "values":
|
|
log.Infof("Parsing values")
|
|
|
|
scanner := bufio.NewScanner(bytes.NewReader(content))
|
|
for scanner.Scan() {
|
|
value := strings.TrimSpace(scanner.Text())
|
|
ret = append(ret, decisionRaw{Value: value})
|
|
}
|
|
|
|
if err := scanner.Err(); err != nil {
|
|
return nil, fmt.Errorf("unable to parse values: '%s'", err)
|
|
}
|
|
case "json":
|
|
log.Infof("Parsing json")
|
|
|
|
if err := json.Unmarshal(content, &ret); err != nil {
|
|
return nil, err
|
|
}
|
|
case "csv":
|
|
log.Infof("Parsing csv")
|
|
|
|
if err := csvutil.Unmarshal(content, &ret); err != nil {
|
|
return nil, fmt.Errorf("unable to parse csv: '%s'", err)
|
|
}
|
|
default:
|
|
return nil, fmt.Errorf("invalid format '%s', expected one of 'json', 'csv', 'values'", format)
|
|
}
|
|
|
|
return ret, nil
|
|
}
|
|
|
|
|
|
func (cli *cliDecisions) runImport(cmd *cobra.Command, args []string) error {
|
|
flags := cmd.Flags()
|
|
|
|
input, err := flags.GetString("input")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
defaultDuration, err := flags.GetString("duration")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if defaultDuration == "" {
|
|
return errors.New("--duration cannot be empty")
|
|
}
|
|
|
|
defaultScope, err := flags.GetString("scope")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if defaultScope == "" {
|
|
return errors.New("--scope cannot be empty")
|
|
}
|
|
|
|
defaultReason, err := flags.GetString("reason")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if defaultReason == "" {
|
|
return errors.New("--reason cannot be empty")
|
|
}
|
|
|
|
defaultType, err := flags.GetString("type")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if defaultType == "" {
|
|
return errors.New("--type cannot be empty")
|
|
}
|
|
|
|
batchSize, err := flags.GetInt("batch")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
format, err := flags.GetString("format")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
var (
|
|
content []byte
|
|
fin *os.File
|
|
)
|
|
|
|
// set format if the file has a json or csv extension
|
|
if format == "" {
|
|
if strings.HasSuffix(input, ".json") {
|
|
format = "json"
|
|
} else if strings.HasSuffix(input, ".csv") {
|
|
format = "csv"
|
|
}
|
|
}
|
|
|
|
if format == "" {
|
|
return errors.New("unable to guess format from file extension, please provide a format with --format flag")
|
|
}
|
|
|
|
if input == "-" {
|
|
fin = os.Stdin
|
|
input = "stdin"
|
|
} else {
|
|
fin, err = os.Open(input)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to open %s: %s", input, err)
|
|
}
|
|
}
|
|
|
|
content, err = io.ReadAll(fin)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to read from %s: %s", input, err)
|
|
}
|
|
|
|
decisionsListRaw, err := parseDecisionList(content, format)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
decisions := make([]*models.Decision, len(decisionsListRaw))
|
|
|
|
for i, d := range decisionsListRaw {
|
|
if d.Value == "" {
|
|
return fmt.Errorf("item %d: missing 'value'", i)
|
|
}
|
|
|
|
if d.Duration == "" {
|
|
d.Duration = defaultDuration
|
|
log.Debugf("item %d: missing 'duration', using default '%s'", i, defaultDuration)
|
|
}
|
|
|
|
if d.Scenario == "" {
|
|
d.Scenario = defaultReason
|
|
log.Debugf("item %d: missing 'reason', using default '%s'", i, defaultReason)
|
|
}
|
|
|
|
if d.Type == "" {
|
|
d.Type = defaultType
|
|
log.Debugf("item %d: missing 'type', using default '%s'", i, defaultType)
|
|
}
|
|
|
|
if d.Scope == "" {
|
|
d.Scope = defaultScope
|
|
log.Debugf("item %d: missing 'scope', using default '%s'", i, defaultScope)
|
|
}
|
|
|
|
decisions[i] = &models.Decision{
|
|
Value: ptr.Of(d.Value),
|
|
Duration: ptr.Of(d.Duration),
|
|
Origin: ptr.Of(types.CscliImportOrigin),
|
|
Scenario: ptr.Of(d.Scenario),
|
|
Type: ptr.Of(d.Type),
|
|
Scope: ptr.Of(d.Scope),
|
|
Simulated: ptr.Of(false),
|
|
}
|
|
}
|
|
|
|
if len(decisions) > 1000 {
|
|
log.Infof("You are about to add %d decisions, this may take a while", len(decisions))
|
|
}
|
|
|
|
for _, chunk := range slicetools.Chunks(decisions, batchSize) {
|
|
log.Debugf("Processing chunk of %d decisions", len(chunk))
|
|
importAlert := models.Alert{
|
|
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
|
Scenario: ptr.Of(fmt.Sprintf("import %s: %d IPs", input, len(chunk))),
|
|
|
|
Message: ptr.Of(""),
|
|
Events: []*models.Event{},
|
|
Source: &models.Source{
|
|
Scope: ptr.Of(""),
|
|
Value: ptr.Of(""),
|
|
},
|
|
StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
|
|
StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
|
|
Capacity: ptr.Of(int32(0)),
|
|
Simulated: ptr.Of(false),
|
|
EventsCount: ptr.Of(int32(len(chunk))),
|
|
Leakspeed: ptr.Of(""),
|
|
ScenarioHash: ptr.Of(""),
|
|
ScenarioVersion: ptr.Of(""),
|
|
Decisions: chunk,
|
|
}
|
|
|
|
_, _, err = Client.Alerts.Add(context.Background(), models.AddAlertsRequest{&importAlert})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
log.Infof("Imported %d decisions", len(decisions))
|
|
|
|
return nil
|
|
}
|
|
|
|
func (cli *cliDecisions) newImportCmd() *cobra.Command {
|
|
cmd := &cobra.Command{
|
|
Use: "import [options]",
|
|
Short: "Import decisions from a file or pipe",
|
|
Long: "expected format:\n" +
|
|
"csv : any of duration,reason,scope,type,value, with a header line\n" +
|
|
"json :" + "`{" + `"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"` + "}`",
|
|
Args: cobra.NoArgs,
|
|
DisableAutoGenTag: true,
|
|
Example: `decisions.csv:
|
|
duration,scope,value
|
|
24h,ip,1.2.3.4
|
|
|
|
$ cscli decisions import -i decisions.csv
|
|
|
|
decisions.json:
|
|
[{"duration" : "4h", "scope" : "ip", "type" : "ban", "value" : "1.2.3.4"}]
|
|
|
|
The file format is detected from the extension, but can be forced with the --format option
|
|
which is required when reading from standard input.
|
|
|
|
Raw values, standard input:
|
|
|
|
$ echo "1.2.3.4" | cscli decisions import -i - --format values
|
|
`,
|
|
RunE: cli.runImport,
|
|
}
|
|
|
|
flags := cmd.Flags()
|
|
flags.SortFlags = false
|
|
flags.StringP("input", "i", "", "Input file")
|
|
flags.StringP("duration", "d", "4h", "Decision duration: 1h,4h,30m")
|
|
flags.String("scope", types.Ip, "Decision scope: ip,range,username")
|
|
flags.StringP("reason", "R", "manual", "Decision reason: <scenario-name>")
|
|
flags.StringP("type", "t", "ban", "Decision type: ban,captcha,throttle")
|
|
flags.Int("batch", 0, "Split import in batches of N decisions")
|
|
flags.String("format", "", "Input format: 'json', 'csv' or 'values' (each line is a value, no headers)")
|
|
|
|
cmd.MarkFlagRequired("input")
|
|
|
|
return cmd
|
|
}
|