diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index 721920160..4b766df65 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -56,7 +56,7 @@ cscli hub update # Download list of available configurations from the hub log.Info(v) } cwhub.DisplaySummary() - ListItems([]string{cwhub.PARSERS, cwhub.COLLECTIONS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW}, args, true, false) + ListItems([]string{cwhub.PARSERS, cwhub.COLLECTIONS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW, cwhub.DATA_FILES}, args, true, false) }, } cmdHubList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") diff --git a/cmd/crowdsec-cli/utils.go b/cmd/crowdsec-cli/utils.go index 666deb4f7..70566097d 100644 --- a/cmd/crowdsec-cli/utils.go +++ b/cmd/crowdsec-cli/utils.go @@ -250,22 +250,21 @@ func UpgradeConfig(itemType string, name string, force bool) { continue } + found = true + if v.UpToDate || v.Tainted { + if v.UpToDate { + log.Infof("%s : up-to-date", v.Name) + } + if err = cwhub.DownloadDataIfNeeded(csConfig.Hub, v, force); err != nil { + log.Fatalf("%s : download failed : %v", v.Name, err) + } + } + if !v.Downloaded { log.Warningf("%s : not downloaded, please install.", v.Name) continue } - found = true - if v.UpToDate { - log.Infof("%s : up-to-date", v.Name) - - if !force { - if err = cwhub.DownloadDataIfNeeded(csConfig.Hub, v, false); err != nil { - log.Fatalf("%s : download failed : %v", v.Name, err) - } - continue - } - } v, err = cwhub.DownloadLatest(csConfig.Hub, v, force, true) if err != nil { log.Fatalf("%s : download failed : %v", v.Name, err) diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index 207ea89af..873cdcbcf 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -22,11 +22,12 @@ var PARSERS = "parsers" var PARSERS_OVFLW = "postoverflows" var SCENARIOS = "scenarios" var COLLECTIONS = "collections" -var ItemTypes = []string{PARSERS, PARSERS_OVFLW, SCENARIOS, COLLECTIONS} +var DATA_FILES = "data_files" +var ItemTypes = []string{PARSERS, PARSERS_OVFLW, SCENARIOS, COLLECTIONS, DATA_FILES} var hubIdx map[string]map[string]Item -var RawFileURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s" +var RawFileURLTemplate = "https://raw.githubusercontent.com/sbs2001/hub/%s/%s" var HubBranch = "master" var HubIndexFile = ".index.json" @@ -57,11 +58,10 @@ type Item struct { BelongsToCollections []string `yaml:"belongs_to_collections,omitempty" json:"belongs_to_collections,omitempty"` /*if it's part of collections, track name here*/ /*remote (hub) infos*/ - RemoteURL string `yaml:"remoteURL,omitempty" json:"remoteURL,omitempty"` //the full remote uri of file in http - RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` //the path relative to git ie. /parsers/stage/author/file.yaml - RemoteHash string `yaml:"hash,omitempty" json:"hash,omitempty"` //the meow - Version string `json:"version,omitempty"` //the last version - Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` //the list of existing versions + RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` //the path relative to git ie. /parsers/stage/author/file.yaml + RemoteHash string `yaml:"hash,omitempty" json:"hash,omitempty"` //the meow + Version string `json:"version,omitempty"` //the last version + Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` //the list of existing versions /*local (deployed) infos*/ LocalPath string `yaml:"local_path,omitempty" json:"local_path,omitempty"` //the local path relative to ${CFG_DIR} diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index c131515b4..d9ea6f930 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -383,8 +383,9 @@ func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) { responseBody := "" log.Printf("---> %s", req.URL.Path) + /*FAKE PARSER*/ - if req.URL.Path == "/master/parsers/s01-parse/crowdsecurity/foobar_parser.yaml" { + if strings.HasSuffix(req.URL.Path, "/master/parsers/s01-parse/crowdsecurity/foobar_parser.yaml") { responseBody = `onsuccess: next_stage filter: evt.Parsed.program == 'foobar_parser' name: crowdsecurity/foobar_parser @@ -395,7 +396,7 @@ grok: apply_on: message ` - } else if req.URL.Path == "/master/parsers/s01-parse/crowdsecurity/foobar_subparser.yaml" { + } else if strings.HasSuffix(req.URL.Path, "/master/parsers/s01-parse/crowdsecurity/foobar_subparser.yaml") { responseBody = `onsuccess: next_stage filter: evt.Parsed.program == 'foobar_parser' name: crowdsecurity/foobar_parser @@ -407,19 +408,19 @@ grok: ` /*FAKE SCENARIO*/ - } else if req.URL.Path == "/master/scenarios/crowdsecurity/foobar_scenario.yaml" { + } else if strings.HasSuffix(req.URL.Path, "/master/scenarios/crowdsecurity/foobar_scenario.yaml") { responseBody = `filter: true name: crowdsecurity/foobar_scenario` /*FAKE COLLECTIONS*/ - } else if req.URL.Path == "/master/collections/crowdsecurity/foobar.yaml" { + } else if strings.HasSuffix(req.URL.Path, "/master/collections/crowdsecurity/foobar.yaml") { responseBody = ` blah: blalala qwe: jejwejejw` - } else if req.URL.Path == "/master/collections/crowdsecurity/foobar_subcollection.yaml" { + } else if strings.HasSuffix(req.URL.Path, "/master/collections/crowdsecurity/foobar_subcollection.yaml") { responseBody = ` blah: blalala qwe: jejwejejw` - } else if req.URL.Path == "/master/.index.json" { + } else if strings.HasSuffix(req.URL.Path, "/master/.index.json") { responseBody = `{ "collections": { diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go index 57879e2ca..d28777e2a 100644 --- a/pkg/cwhub/download.go +++ b/pkg/cwhub/download.go @@ -3,15 +3,17 @@ package cwhub import ( "bytes" "crypto/sha256" + "io" "path" "path/filepath" + "sort" //"errors" "github.com/pkg/errors" + "gopkg.in/yaml.v2" //"errors" "fmt" - "io" "io/ioutil" "net/http" "os" @@ -20,7 +22,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/types" log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" ) func UpdateHubIdx(hub *csconfig.Hub) error { @@ -79,7 +80,6 @@ func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) { func DownloadLatest(hub *csconfig.Hub, target Item, overwrite bool, updateOnly bool) (Item, error) { var err error - log.Debugf("Downloading %s %s", target.Type, target.Name) if target.Type == COLLECTIONS { var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} for idx, ptr := range tmp { @@ -143,10 +143,19 @@ func DownloadItem(hub *csconfig.Hub, target Item, overwrite bool) (Item, error) return target, nil } if target.UpToDate { - log.Debugf("%s : up-to-date, not updated", target.Name) // We still have to check if data files are present + log.Debugf("%s : up-to-date, not updated", target.Name) + data, err := os.ReadFile(target.LocalPath) + if err != nil { + return target, err + } + if err := downloadData(dataFolder, target.Author, overwrite, bytes.NewReader(data)); err != nil { + return target, errors.Wrapf(err, "while downloading data for %s", target.FileName) + } + return target, nil } } + log.Debugf("Downloading %s %s", target.Type, target.Name) req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil) if err != nil { return target, errors.Wrap(err, fmt.Sprintf("while downloading %s", req.URL.String())) @@ -214,7 +223,7 @@ func DownloadItem(hub *csconfig.Hub, target Item, overwrite bool) (Item, error) target.Tainted = false target.UpToDate = true - if err = downloadData(dataFolder, overwrite, bytes.NewReader(body)); err != nil { + if err = downloadData(dataFolder, target.Author, overwrite, bytes.NewReader(body)); err != nil { return target, errors.Wrapf(err, "while downloading data for %s", target.FileName) } @@ -232,16 +241,15 @@ func DownloadDataIfNeeded(hub *csconfig.Hub, target Item, force bool) error { if itemFile, err = os.Open(itemFilePath); err != nil { return errors.Wrapf(err, "while opening %s", itemFilePath) } - if err = downloadData(dataFolder, force, itemFile); err != nil { + if err = downloadData(dataFolder, target.Author, force, itemFile); err != nil { return errors.Wrapf(err, "while downloading data for %s", itemFilePath) } return nil } -func downloadData(dataFolder string, force bool, reader io.Reader) error { +func downloadData(dataFolder string, parentItemAuthor string, force bool, reader io.Reader) error { var err error dec := yaml.NewDecoder(reader) - for { data := &types.DataSet{} err = dec.Decode(data) @@ -252,20 +260,59 @@ func downloadData(dataFolder string, force bool, reader io.Reader) error { break } - download := false - if !force { - for _, dataS := range data.Data { - if _, err := os.Stat(path.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) { - download = true + for _, dataS := range data.Data { + download := false + dfPath := path.Join(dataFolder, dataS.DestPath) + dataFileName := strings.Split(dataS.DestPath, ".")[0] + _, downloadFromHub := hubIdx[DATA_FILES][dataFileName] + if downloadFromHub { + dataS.SourceURL = fmt.Sprintf(RawFileURLTemplate, HubBranch, hubIdx[DATA_FILES][dataFileName].RemotePath) + } + + if _, err := os.Stat(dfPath); os.IsNotExist(err) { + download = true + } else if downloadFromHub { + sha, err := getSHA256(dfPath) + if err != nil { + return err + } + download = dataFileHasUpdates(sha, dataFileName) + } + + log.Infof("%v has updates=%v", dataFileName, download) + + if download || force { + err = types.GetData(dataS, dataFolder) + if err != nil { + return errors.Wrap(err, "while getting data") } } - } - if download || force { - err = types.GetData(data.Data, dataFolder) - if err != nil { - return errors.Wrap(err, "while getting data") - } + } } + return nil } + +// Checks if the provided data file is latest. Only files which are available in hub should +// be checked for. +func dataFileHasUpdates(fileSha string, dataFileName string) bool { + dataItem := hubIdx[DATA_FILES][dataFileName] + versions := make([]string, 0, len(dataItem.Versions)) + for k := range dataItem.Versions { + versions = append(versions, k) + } + sort.Sort(sort.Reverse(sort.StringSlice(versions))) + for i, version := range versions { + if fileSha != dataItem.Versions[version].Digest { + continue + } + log.Debugf("data file %s matched sha with version %s", dataFileName, version) + if i != 0 { + log.Debugf("data file %s is outdated, updating to version %s", dataFileName, versions[0]) + return true + } + break + } + return false +} diff --git a/pkg/cwhub/download_test.go b/pkg/cwhub/download_test.go index 156c41322..b2ee5f7b5 100644 --- a/pkg/cwhub/download_test.go +++ b/pkg/cwhub/download_test.go @@ -40,3 +40,29 @@ func TestDownloadHubIdx(t *testing.T) { RawFileURLTemplate = back fmt.Printf("->%+v", ret) } + +func TestDataFileIsLatest(t *testing.T) { + dataFileName := "crowdsecurity/sensitive-files" + hubIdx = map[string]map[string]Item{ + "data_files": { + "crowdsecurity/sensitive-files": { + Versions: map[string]ItemVersion{ + "0.1": {Digest: "1"}, + "0.2": {Digest: "2"}, + }, + }, + }, + } + if dataFileHasUpdates("1", dataFileName) { + log.Errorf(`expected dataFileIsLatest("1", %s) = true found false `, dataFileName) + } + + if !dataFileHasUpdates("2", dataFileName) { + log.Errorf(`expected dataFileIsLatest("2", %s) = false found true `, dataFileName) + } + + // data file is tainted + if dataFileHasUpdates("3", dataFileName) { + log.Errorf(`expected dataFileIsLatest("3", %s) = false found true `, dataFileName) + } +} diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/loader.go index 819e5dc87..8682ff266 100644 --- a/pkg/cwhub/loader.go +++ b/pkg/cwhub/loader.go @@ -21,10 +21,10 @@ import ( ) /*the walk/parser_visit function can't receive extra args*/ -var hubdir, installdir string +var hubdir, installdir, datadir string +// TODO: Break this function into smaller functions. func parser_visit(path string, f os.FileInfo, err error) error { - var target Item var local bool var hubpath string @@ -42,16 +42,17 @@ func parser_visit(path string, f os.FileInfo, err error) error { if f == nil || f.IsDir() { return nil } - //we only care about yaml files - if !strings.HasSuffix(f.Name(), ".yaml") && !strings.HasSuffix(f.Name(), ".yml") { + // yamls -> collections, parsers, overflows etc. txt, mmdb -> data files + if !strings.HasSuffix(f.Name(), ".yaml") && !strings.HasSuffix(f.Name(), ".yml") && !strings.HasSuffix(f.Name(), ".txt") && !strings.HasSuffix(f.Name(), ".mmdb") { return nil } subs := strings.Split(path, "/") - log.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubdir, installdir) + log.Tracef("path:%s, hubdir:%s, installdir:%s datadir%s", path, hubdir, installdir, datadir) /*we're in hub (~/.hub/hub/)*/ - if strings.HasPrefix(path, hubdir) { + + hubDirSetter := func() { log.Tracef("in hub dir") inhub = true //.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml @@ -64,7 +65,18 @@ func parser_visit(path string, f os.FileInfo, err error) error { fauthor = subs[len(subs)-2] stage = subs[len(subs)-3] ftype = subs[len(subs)-4] - } else if strings.HasPrefix(path, installdir) { /*we're in install /etc/crowdsec//... */ + } + + dataDirSetter := func() { + log.Tracef("in data dir") + fauthor = "" + fname = subs[len(subs)-1] + stage = "" + ftype = DATA_FILES + fauthor = "" + } + + installDirSetter := func() { log.Tracef("in install dir") if len(subs) < 3 { log.Fatalf("path is too short : %s (%d)", path, len(subs)) @@ -76,23 +88,42 @@ func parser_visit(path string, f os.FileInfo, err error) error { fname = subs[len(subs)-1] stage = subs[len(subs)-2] ftype = subs[len(subs)-3] - fauthor = "" - } else { - return fmt.Errorf("File '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubdir, installdir) } + setterByPath := map[string]func(){ + installdir: installDirSetter, + hubdir: hubDirSetter, + datadir: dataDirSetter, + } + + paths := []string{installdir, hubdir, datadir} + sort.Slice(paths, func(i, j int) bool { + return len(paths[i]) > len(paths[j]) + }) + foundMatch := false + for _, p := range paths { + if strings.HasPrefix(path, p) { + setterByPath[p]() + foundMatch = true + break + } + } + if !foundMatch { + return fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubdir, installdir) + } log.Tracef("stage:%s ftype:%s", stage, ftype) //log.Printf("%s -> name:%s stage:%s", path, fname, stage) + + // correct the stage and type for non-stage stuff. if stage == SCENARIOS { ftype = SCENARIOS stage = "" } else if stage == COLLECTIONS { ftype = COLLECTIONS stage = "" - } else if ftype != PARSERS && ftype != PARSERS_OVFLW /*its a PARSER / PARSER_OVFLW with a stage */ { + } else if ftype != PARSERS && ftype != PARSERS_OVFLW && ftype != DATA_FILES { /*its a PARSER / PARSER_OVFLW with a stage */ return fmt.Errorf("unknown configuration type for file '%s'", path) } - log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, stage, ftype) /* @@ -100,7 +131,7 @@ func parser_visit(path string, f os.FileInfo, err error) error { /etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml when the collection is installed, both files are created */ - //non symlinks are local user files or hub files + //non symlinks are local user files or hub files or data files if f.Mode()&os.ModeSymlink == 0 { local = true log.Tracef("%s isn't a symlink", path) @@ -122,8 +153,8 @@ func parser_visit(path string, f os.FileInfo, err error) error { log.Tracef("%s points to %s", path, hubpath) } - //if it's not a symlink and not in hub, it's a local file, don't bother - if local && !inhub { + //if it's not a symlink and not in hub nor it is a data file. Don't bother checking this with index + if local && !inhub && ftype != DATA_FILES { log.Tracef("%s is a local file, skip", path) skippedLocal++ // log.Printf("local scenario, skip.") @@ -144,7 +175,7 @@ func parser_visit(path string, f os.FileInfo, err error) error { log.Tracef("check [%s] of %s", fname, ftype) match := false - for k, v := range hubIdx[ftype] { + for k, v := range hubIdx[ftype] { // eg ftype = "collections", k = crowdsecurity/nginx, v is an Item struct log.Tracef("check [%s] vs [%s] : %s", fname, v.RemotePath, ftype+"/"+stage+"/"+fname+".yaml") if fname != v.FileName { log.Tracef("%s != %s (filename)", fname, v.FileName) @@ -152,6 +183,7 @@ func parser_visit(path string, f os.FileInfo, err error) error { } //wrong stage if v.Stage != stage { + log.Tracef("%s != %s (stage)", v.Stage, stage) continue } /*if we are walking hub dir, just mark present files as downloaded*/ @@ -168,14 +200,6 @@ func parser_visit(path string, f os.FileInfo, err error) error { log.Tracef("marking %s as downloaded", v.Name) v.Downloaded = true } - } else { - //wrong file - /////.yaml - if !strings.HasSuffix(hubpath, v.RemotePath) { - //log.Printf("wrong file %s %s", hubpath, spew.Sdump(v)) - - continue - } } sha, err := getSHA256(path) if err != nil { @@ -305,11 +329,18 @@ func CollecDepsCheck(v *Item) error { func SyncDir(hub *csconfig.Hub, dir string) (error, []string) { hubdir = hub.HubDir installdir = hub.ConfigDir + datadir = hub.DataDir warnings := []string{} - /*For each, scan PARSERS, PARSERS_OVFLW, SCENARIOS and COLLECTIONS last*/ + /*For each, scan PARSERS, PARSERS_OVFLW, DATA_FILES, SCENARIOS and COLLECTIONS last*/ for _, scan := range ItemTypes { - cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan)) + var cpath string + var err error + if scan == DATA_FILES { + cpath, err = filepath.Abs(hub.DataDir) + } else { + cpath, err = filepath.Abs(fmt.Sprintf("%s/%s", dir, scan)) + } if err != nil { log.Errorf("failed %s : %s", cpath, err) } @@ -317,24 +348,24 @@ func SyncDir(hub *csconfig.Hub, dir string) (error, []string) { if err != nil { return err, warnings } - } for k, v := range hubIdx[COLLECTIONS] { - if v.Installed { - versStat := GetVersionStatus(&v) - if versStat == 0 { //latest - if err := CollecDepsCheck(&v); err != nil { - warnings = append(warnings, fmt.Sprintf("dependency of %s : %s", v.Name, err)) - hubIdx[COLLECTIONS][k] = v - } - } else if versStat == 1 { //not up-to-date - warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", v.Name, v.LocalVersion, v.Version)) - } else { //version is higher than the highest available from hub? - warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", v.Name, v.LocalVersion, v.Version)) - } - log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", v.Name, semver.Compare("v"+v.Version, "v"+v.LocalVersion), v.LocalVersion, v.Version, v.Versions) + if !v.Installed { + continue } + versStat := GetVersionStatus(&v) + if versStat == 0 { //latest + if err := CollecDepsCheck(&v); err != nil { + warnings = append(warnings, fmt.Sprintf("dependency of %s : %s", v.Name, err)) + hubIdx[COLLECTIONS][k] = v + } + } else if versStat == 1 { //not up-to-date + warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", v.Name, v.LocalVersion, v.Version)) + } else { //version is higher than the highest available from hub? + warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", v.Name, v.LocalVersion, v.Version)) + } + log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", v.Name, semver.Compare("v"+v.Version, "v"+v.LocalVersion), v.LocalVersion, v.Version, v.Versions) } return nil, warnings } diff --git a/pkg/types/dataset.go b/pkg/types/dataset.go index de3a215da..c9f79b668 100644 --- a/pkg/types/dataset.go +++ b/pkg/types/dataset.go @@ -6,6 +6,7 @@ import ( "net/http" "os" "path" + "path/filepath" log "github.com/sirupsen/logrus" ) @@ -42,6 +43,10 @@ func downloadFile(url string, destPath string) error { return fmt.Errorf("download response 'HTTP %d' : %s", resp.StatusCode, string(body)) } + if err := os.MkdirAll(filepath.Dir(destPath), 0666); err != nil { + return err + } + file, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return err @@ -60,14 +65,12 @@ func downloadFile(url string, destPath string) error { return nil } -func GetData(data []*DataSource, dataDir string) error { - for _, dataS := range data { - destPath := path.Join(dataDir, dataS.DestPath) - log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath) - err := downloadFile(dataS.SourceURL, destPath) - if err != nil { - return err - } +func GetData(dataS *DataSource, dataDir string) error { + destPath := path.Join(dataDir, dataS.DestPath) + log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath) + err := downloadFile(dataS.SourceURL, destPath) + if err != nil { + return err } return nil