diff --git a/cmd/crowdsec-cli/papi.go b/cmd/crowdsec-cli/papi.go index ccd9cebbd..9951d5db2 100644 --- a/cmd/crowdsec-cli/papi.go +++ b/cmd/crowdsec-cli/papi.go @@ -51,7 +51,7 @@ func NewPapiStatusCmd() *cobra.Command { log.Fatalf("unable to initialize database client : %s", err) } - apic, err := apiserver.NewAPIC(csConfig.API.Server.OnlineClient, dbClient, csConfig.API.Server.ConsoleConfig) + apic, err := apiserver.NewAPIC(csConfig.API.Server.OnlineClient, dbClient, csConfig.API.Server.ConsoleConfig, csConfig.API.Server.CapiWhitelists) if err != nil { log.Fatalf("unable to initialize API client : %s", err) @@ -101,7 +101,7 @@ func NewPapiSyncCmd() *cobra.Command { log.Fatalf("unable to initialize database client : %s", err) } - apic, err := apiserver.NewAPIC(csConfig.API.Server.OnlineClient, dbClient, csConfig.API.Server.ConsoleConfig) + apic, err := apiserver.NewAPIC(csConfig.API.Server.OnlineClient, dbClient, csConfig.API.Server.ConsoleConfig, csConfig.API.Server.CapiWhitelists) if err != nil { log.Fatalf("unable to initialize API client : %s", err) diff --git a/pkg/apiserver/apic.go b/pkg/apiserver/apic.go index d4dd4f86a..e811f6004 100644 --- a/pkg/apiserver/apic.go +++ b/pkg/apiserver/apic.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math/rand" + "net" "net/http" "net/url" "strconv" @@ -60,6 +61,7 @@ type apic struct { credentials *csconfig.ApiCredentialsCfg scenarioList []string consoleConfig *csconfig.ConsoleConfig + whitelists *csconfig.CapiWhitelist } // randomDuration returns a duration value between d-delta and d+delta @@ -149,7 +151,7 @@ func alertToSignal(alert *models.Alert, scenarioTrust string, shareContext bool) return signal } -func NewAPIC(config *csconfig.OnlineApiClientCfg, dbClient *database.Client, consoleConfig *csconfig.ConsoleConfig) (*apic, error) { +func NewAPIC(config *csconfig.OnlineApiClientCfg, dbClient *database.Client, consoleConfig *csconfig.ConsoleConfig, apicWhitelist *csconfig.CapiWhitelist) (*apic, error) { var err error ret := &apic{ @@ -169,6 +171,7 @@ func NewAPIC(config *csconfig.OnlineApiClientCfg, dbClient *database.Client, con pushIntervalFirst: randomDuration(pushIntervalDefault, pushIntervalDelta), metricsInterval: metricsIntervalDefault, metricsIntervalFirst: randomDuration(metricsIntervalDefault, metricsIntervalDelta), + whitelists: apicWhitelist, } password := strfmt.Password(config.Credentials.Password) @@ -573,6 +576,9 @@ func (a *apic) PullTop() error { // create one alert for community blocklist using the first decision decisions := a.apiClient.Decisions.GetDecisionsFromGroups(data.New) + //apply APIC specific whitelists + decisions = a.ApplyApicWhitelists(decisions) + alert := createAlertForDecision(decisions[0]) alertsFromCapi := []*models.Alert{alert} alertsFromCapi = fillAlertsWithDecisions(alertsFromCapi, decisions, add_counters) @@ -589,6 +595,47 @@ func (a *apic) PullTop() error { return nil } +func (a *apic) ApplyApicWhitelists(decisions []*models.Decision) []*models.Decision { + if a.whitelists == nil { + return decisions + } + //deal with CAPI whitelists for fire. We want to avoid having a second list, so we shrink in place + outIdx := 0 + for _, decision := range decisions { + if decision.Value == nil { + continue + } + skip := false + ipval := net.ParseIP(*decision.Value) + for _, cidr := range a.whitelists.Cidrs { + if skip { + break + } + if cidr.Contains(ipval) { + log.Infof("%s from %s is whitelisted by %s", *decision.Value, *decision.Scenario, cidr.String()) + skip = true + } + } + for _, ip := range a.whitelists.Ips { + if skip { + break + } + if ip != nil && ip.Equal(ipval) { + log.Infof("%s from %s is whitelisted by %s", *decision.Value, *decision.Scenario, ip.String()) + skip = true + } + } + if !skip { + decisions[outIdx] = decision + outIdx++ + } + + } + //shrink the list, those are deleted items + decisions = decisions[:outIdx] + return decisions +} + func (a *apic) SaveAlerts(alertsFromCapi []*models.Alert, add_counters map[string]map[string]int, delete_counters map[string]map[string]int) error { for idx, alert := range alertsFromCapi { alertsFromCapi[idx] = setAlertScenario(add_counters, delete_counters, alert) @@ -690,6 +737,8 @@ func (a *apic) UpdateBlocklists(links *modelscapi.GetDecisionsStreamResponseLink log.Infof("blocklist %s has no decisions", *blocklist.Name) continue } + //apply APIC specific whitelists + decisions = a.ApplyApicWhitelists(decisions) alert := createAlertForDecision(decisions[0]) alertsFromCapi := []*models.Alert{alert} alertsFromCapi = fillAlertsWithDecisions(alertsFromCapi, decisions, add_counters) diff --git a/pkg/apiserver/apic_test.go b/pkg/apiserver/apic_test.go index fec7d8182..df3d2e965 100644 --- a/pkg/apiserver/apic_test.go +++ b/pkg/apiserver/apic_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "net" "net/http" "net/url" "os" @@ -232,7 +233,7 @@ func TestNewAPIC(t *testing.T) { ), )) tc.action() - _, err := NewAPIC(testConfig, tc.args.dbClient, tc.args.consoleConfig) + _, err := NewAPIC(testConfig, tc.args.dbClient, tc.args.consoleConfig, nil) cstest.RequireErrorContains(t, err, tc.expectedErr) }) } @@ -527,6 +528,188 @@ func TestFillAlertsWithDecisions(t *testing.T) { } } +func TestAPICWhitelists(t *testing.T) { + api := getAPIC(t) + //one whitelist on IP, one on CIDR + api.whitelists = &csconfig.CapiWhitelist{} + ipwl1 := "9.2.3.4" + ip := net.ParseIP(ipwl1) + api.whitelists.Ips = append(api.whitelists.Ips, ip) + ipwl1 = "7.2.3.4" + ip = net.ParseIP(ipwl1) + api.whitelists.Ips = append(api.whitelists.Ips, ip) + cidrwl1 := "13.2.3.0/24" + _, tnet, err := net.ParseCIDR(cidrwl1) + if err != nil { + t.Fatalf("unable to parse cidr : %s", err) + } + api.whitelists.Cidrs = append(api.whitelists.Cidrs, tnet) + cidrwl1 = "11.2.3.0/24" + _, tnet, err = net.ParseCIDR(cidrwl1) + if err != nil { + t.Fatalf("unable to parse cidr : %s", err) + } + api.whitelists.Cidrs = append(api.whitelists.Cidrs, tnet) + api.dbClient.Ent.Decision.Create(). + SetOrigin(types.CAPIOrigin). + SetType("ban"). + SetValue("9.9.9.9"). + SetScope("Ip"). + SetScenario("crowdsecurity/ssh-bf"). + SetUntil(time.Now().Add(time.Hour)). + ExecX(context.Background()) + assertTotalDecisionCount(t, api.dbClient, 1) + assertTotalValidDecisionCount(t, api.dbClient, 1) + httpmock.Activate() + defer httpmock.DeactivateAndReset() + httpmock.RegisterResponder("GET", "http://api.crowdsec.net/api/decisions/stream", httpmock.NewBytesResponder( + 200, jsonMarshalX( + modelscapi.GetDecisionsStreamResponse{ + Deleted: modelscapi.GetDecisionsStreamResponseDeleted{ + &modelscapi.GetDecisionsStreamResponseDeletedItem{ + Decisions: []string{ + "9.9.9.9", // This is already present in DB + "9.1.9.9", // This not present in DB + }, + Scope: types.StrPtr("Ip"), + }, // This is already present in DB + }, + New: modelscapi.GetDecisionsStreamResponseNew{ + &modelscapi.GetDecisionsStreamResponseNewItem{ + Scenario: types.StrPtr("crowdsecurity/test1"), + Scope: types.StrPtr("Ip"), + Decisions: []*modelscapi.GetDecisionsStreamResponseNewItemDecisionsItems0{ + { + Value: types.StrPtr("13.2.3.4"), //wl by cidr + Duration: types.StrPtr("24h"), + }, + }, + }, + + &modelscapi.GetDecisionsStreamResponseNewItem{ + Scenario: types.StrPtr("crowdsecurity/test1"), + Scope: types.StrPtr("Ip"), + Decisions: []*modelscapi.GetDecisionsStreamResponseNewItemDecisionsItems0{ + { + Value: types.StrPtr("2.2.3.4"), + Duration: types.StrPtr("24h"), + }, + }, + }, + &modelscapi.GetDecisionsStreamResponseNewItem{ + Scenario: types.StrPtr("crowdsecurity/test2"), + Scope: types.StrPtr("Ip"), + Decisions: []*modelscapi.GetDecisionsStreamResponseNewItemDecisionsItems0{ + { + Value: types.StrPtr("13.2.3.5"), //wl by cidr + Duration: types.StrPtr("24h"), + }, + }, + }, // These two are from community list. + &modelscapi.GetDecisionsStreamResponseNewItem{ + Scenario: types.StrPtr("crowdsecurity/test1"), + Scope: types.StrPtr("Ip"), + Decisions: []*modelscapi.GetDecisionsStreamResponseNewItemDecisionsItems0{ + { + Value: types.StrPtr("6.2.3.4"), + Duration: types.StrPtr("24h"), + }, + }, + }, + &modelscapi.GetDecisionsStreamResponseNewItem{ + Scenario: types.StrPtr("crowdsecurity/test1"), + Scope: types.StrPtr("Ip"), + Decisions: []*modelscapi.GetDecisionsStreamResponseNewItemDecisionsItems0{ + { + Value: types.StrPtr("9.2.3.4"), //wl by ip + Duration: types.StrPtr("24h"), + }, + }, + }, + }, + Links: &modelscapi.GetDecisionsStreamResponseLinks{ + Blocklists: []*modelscapi.BlocklistLink{ + { + URL: types.StrPtr("http://api.crowdsec.net/blocklist1"), + Name: types.StrPtr("blocklist1"), + Scope: types.StrPtr("Ip"), + Remediation: types.StrPtr("ban"), + Duration: types.StrPtr("24h"), + }, + { + URL: types.StrPtr("http://api.crowdsec.net/blocklist2"), + Name: types.StrPtr("blocklist2"), + Scope: types.StrPtr("Ip"), + Remediation: types.StrPtr("ban"), + Duration: types.StrPtr("24h"), + }, + }, + }, + }, + ), + )) + httpmock.RegisterResponder("GET", "http://api.crowdsec.net/blocklist1", httpmock.NewStringResponder( + 200, "1.2.3.6", + )) + httpmock.RegisterResponder("GET", "http://api.crowdsec.net/blocklist2", httpmock.NewStringResponder( + 200, "1.2.3.7", + )) + url, err := url.ParseRequestURI("http://api.crowdsec.net/") + require.NoError(t, err) + + apic, err := apiclient.NewDefaultClient( + url, + "/api", + fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + nil, + ) + require.NoError(t, err) + + api.apiClient = apic + err = api.PullTop() + require.NoError(t, err) + + assertTotalDecisionCount(t, api.dbClient, 5) //2 from FIRE + 2 from bl + 1 existing + assertTotalValidDecisionCount(t, api.dbClient, 4) + assertTotalAlertCount(t, api.dbClient, 3) // 2 for list sub , 1 for community list. + alerts := api.dbClient.Ent.Alert.Query().AllX(context.Background()) + validDecisions := api.dbClient.Ent.Decision.Query().Where( + decision.UntilGT(time.Now())). + AllX(context.Background()) + + decisionScenarioFreq := make(map[string]int) + decisionIp := make(map[string]int) + + alertScenario := make(map[string]int) + + for _, alert := range alerts { + alertScenario[alert.SourceScope]++ + } + assert.Equal(t, 3, len(alertScenario)) + assert.Equal(t, 1, alertScenario[SCOPE_CAPI_ALIAS_ALIAS]) + assert.Equal(t, 1, alertScenario["lists:blocklist1"]) + assert.Equal(t, 1, alertScenario["lists:blocklist2"]) + + for _, decisions := range validDecisions { + decisionScenarioFreq[decisions.Scenario]++ + decisionIp[decisions.Value]++ + } + assert.Equal(t, 1, decisionIp["2.2.3.4"], 1) + assert.Equal(t, 1, decisionIp["6.2.3.4"], 1) + if _, ok := decisionIp["13.2.3.4"]; ok { + t.Errorf("13.2.3.4 is whitelisted") + } + if _, ok := decisionIp["13.2.3.5"]; ok { + t.Errorf("13.2.3.5 is whitelisted") + } + if _, ok := decisionIp["9.2.3.4"]; ok { + t.Errorf("9.2.3.4 is whitelisted") + } + assert.Equal(t, 1, decisionScenarioFreq["blocklist1"], 1) + assert.Equal(t, 1, decisionScenarioFreq["blocklist2"], 1) + assert.Equal(t, 2, decisionScenarioFreq["crowdsecurity/test1"], 2) +} + func TestAPICPullTop(t *testing.T) { api := getAPIC(t) api.dbClient.Ent.Decision.Create(). diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index bc244916a..73758eae7 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -217,7 +217,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) { if config.OnlineClient != nil && config.OnlineClient.Credentials != nil { log.Printf("Loading CAPI manager") - apiClient, err = NewAPIC(config.OnlineClient, dbClient, config.ConsoleConfig) + apiClient, err = NewAPIC(config.OnlineClient, dbClient, config.ConsoleConfig, config.CapiWhitelists) if err != nil { return &APIServer{}, err } diff --git a/pkg/csconfig/api.go b/pkg/csconfig/api.go index 80a235e2c..bcc3c5b93 100644 --- a/pkg/csconfig/api.go +++ b/pkg/csconfig/api.go @@ -173,6 +173,11 @@ func toValidCIDR(ip string) string { return ip + "/32" } +type CapiWhitelist struct { + Ips []net.IP `yaml:"ips,omitempty"` + Cidrs []*net.IPNet `yaml:"cidrs,omitempty"` +} + /*local api service configuration*/ type LocalApiServerCfg struct { Enable *bool `yaml:"enable"` @@ -196,6 +201,8 @@ type LocalApiServerCfg struct { TrustedIPs []string `yaml:"trusted_ips,omitempty"` PapiLogLevel *log.Level `yaml:"papi_log_level"` DisableRemoteLapiRegistration bool `yaml:"disable_remote_lapi_registration,omitempty"` + CapiWhitelistsPath string `yaml:"capi_whitelists_path,omitempty"` + CapiWhitelists *CapiWhitelist `yaml:"-"` } type TLSCfg struct { @@ -242,6 +249,11 @@ func (c *Config) LoadAPIServer() error { if err := c.LoadDBConfig(); err != nil { return err } + + if err := c.API.Server.LoadCapiWhitelists(); err != nil { + return err + } + } else { log.Warning("crowdsec local API is disabled") c.DisableAPI = true @@ -306,6 +318,49 @@ func (c *Config) LoadAPIServer() error { return nil } +// we cannot unmarshal to type net.IPNet, so we need to do it manually +type capiWhitelists struct { + Ips []string `yaml:"ips"` + Cidrs []string `yaml:"cidrs"` +} + +func (s *LocalApiServerCfg) LoadCapiWhitelists() error { + if s.CapiWhitelistsPath == "" { + return nil + } + if _, err := os.Stat(s.CapiWhitelistsPath); os.IsNotExist(err) { + return fmt.Errorf("capi whitelist file '%s' does not exist", s.CapiWhitelistsPath) + } + fd, err := os.Open(s.CapiWhitelistsPath) + if err != nil { + return fmt.Errorf("unable to open capi whitelist file '%s': %s", s.CapiWhitelistsPath, err) + } + + var fromCfg capiWhitelists + s.CapiWhitelists = &CapiWhitelist{} + + defer fd.Close() + decoder := yaml.NewDecoder(fd) + if err := decoder.Decode(&fromCfg); err != nil { + return fmt.Errorf("while parsing capi whitelist file '%s': %s", s.CapiWhitelistsPath, err) + } + for _, v := range fromCfg.Ips { + ip := net.ParseIP(v) + if ip == nil { + return fmt.Errorf("unable to parse ip whitelist '%s'", v) + } + s.CapiWhitelists.Ips = append(s.CapiWhitelists.Ips, ip) + } + for _, v := range fromCfg.Cidrs { + _, tnet, err := net.ParseCIDR(v) + if err != nil { + return fmt.Errorf("unable to parse cidr whitelist '%s' : %v.", v, err) + } + s.CapiWhitelists.Cidrs = append(s.CapiWhitelists.Cidrs, tnet) + } + return nil +} + func (c *Config) LoadAPIClient() error { if c.API == nil || c.API.Client == nil || c.API.Client.CredentialsFilePath == "" || c.DisableAgent { return fmt.Errorf("no API client section in configuration")