package cwhub import ( "bytes" "crypto/sha256" "errors" "fmt" "io" "net/http" "os" "path/filepath" "strings" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) var ErrIndexNotFound = fmt.Errorf("index not found") // InitHubUpdate is like InitHub but downloads and updates the index instead of reading from the disk // It is used to inizialize the hub when there is no index file yet func InitHubUpdate(cfg *csconfig.HubCfg) (*Hub, error) { if cfg == nil { return nil, fmt.Errorf("no configuration found for hub") } bidx, err := DownloadHubIdx(cfg.HubIndexFile) if err != nil { return nil, fmt.Errorf("failed to download index: %w", err) } ret, err := ParseIndex(bidx) if err != nil { if !errors.Is(err, ErrMissingReference) { return nil, fmt.Errorf("failed to read index: %w", err) } } theHub = &Hub{ Items: ret, cfg: cfg, } if _, err := theHub.LocalSync(); err != nil { return nil, fmt.Errorf("failed to sync: %w", err) } return theHub, nil } // DownloadHubIdx downloads the latest version of the index and returns the content func DownloadHubIdx(indexPath string) ([]byte, error) { log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile)) req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil) if err != nil { return nil, fmt.Errorf("failed to build request for hub index: %w", err) } resp, err := http.DefaultClient.Do(req) if err != nil { return nil, fmt.Errorf("failed http request for hub index: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { if resp.StatusCode == http.StatusNotFound { return nil, ErrIndexNotFound } return nil, fmt.Errorf("bad http code %d while requesting %s", resp.StatusCode, req.URL.String()) } body, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("failed to read request answer for hub index: %w", err) } oldContent, err := os.ReadFile(indexPath) if err != nil { if !os.IsNotExist(err) { log.Warningf("failed to read hub index: %s", err) } } else if bytes.Equal(body, oldContent) { log.Info("hub index is up to date") // write it anyway, can't hurt } file, err := os.OpenFile(indexPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) if err != nil { return nil, fmt.Errorf("while opening hub index file: %w", err) } defer file.Close() wsize, err := file.Write(body) if err != nil { return nil, fmt.Errorf("while writing hub index file: %w", err) } log.Infof("Wrote new %d bytes index to %s", wsize, indexPath) return body, nil } // DownloadLatest will download the latest version of Item to the tdir directory func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) error { var err error log.Debugf("Downloading %s %s", target.Type, target.Name) if target.Type != COLLECTIONS { if !target.Installed && updateOnly && target.Downloaded { log.Debugf("skipping upgrade of %s : not installed", target.Name) return nil } return h.DownloadItem(target, overwrite) } // collection var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.WaapRules, target.WaapConfigs, target.Collections} for idx, ptr := range tmp { ptrtype := ItemTypes[idx] for _, p := range ptr { val, ok := h.Items[ptrtype][p] if !ok { return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name) } if !val.Installed && updateOnly && val.Downloaded { log.Debugf("skipping upgrade of %s : not installed", target.Name) continue } log.Debugf("Download %s sub-item : %s %s (%t -> %t)", target.Name, ptrtype, p, target.Installed, updateOnly) //recurse as it's a collection if ptrtype == COLLECTIONS { log.Tracef("collection, recurse") err = h.DownloadLatest(&val, overwrite, updateOnly) if err != nil { return fmt.Errorf("while downloading %s: %w", val.Name, err) } } downloaded := val.Downloaded err = h.DownloadItem(&val, overwrite) if err != nil { return fmt.Errorf("while downloading %s: %w", val.Name, err) } // We need to enable an item when it has been added to a collection since latest release of the collection. // We check if val.Downloaded is false because maybe the item has been disabled by the user. if !val.Installed && !downloaded { if err = h.EnableItem(&val); err != nil { return fmt.Errorf("enabling '%s': %w", val.Name, err) } } h.Items[ptrtype][p] = val } } err = h.DownloadItem(target, overwrite) if err != nil { return fmt.Errorf("failed to download item: %w", err) } return nil } func (h *Hub) DownloadItem(target *Item, overwrite bool) error { tdir := h.cfg.HubDir // if user didn't --force, don't overwrite local, tainted, up-to-date files if !overwrite { if target.Tainted { log.Debugf("%s : tainted, not updated", target.Name) return nil } if target.UpToDate { // We still have to check if data files are present log.Debugf("%s : up-to-date, not updated", target.Name) } } req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil) if err != nil { return fmt.Errorf("while downloading %s: %w", req.URL.String(), err) } resp, err := http.DefaultClient.Do(req) if err != nil { return fmt.Errorf("while downloading %s: %w", req.URL.String(), err) } if resp.StatusCode != http.StatusOK { return fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String()) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { return fmt.Errorf("while reading %s: %w", req.URL.String(), err) } hash := sha256.New() if _, err = hash.Write(body); err != nil { return fmt.Errorf("while hashing %s: %w", target.Name, err) } meow := fmt.Sprintf("%x", hash.Sum(nil)) if meow != target.Versions[target.Version].Digest { log.Errorf("Downloaded version doesn't match index, please 'hub update'") log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest) return fmt.Errorf("invalid download hash for %s", target.Name) } //all good, install //check if parent dir exists tmpdirs := strings.Split(tdir+"/"+target.RemotePath, "/") parentDir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/") // ensure that target file is within target dir finalPath, err := filepath.Abs(tdir + "/" + target.RemotePath) if err != nil { return fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+target.RemotePath, err) } if !strings.HasPrefix(finalPath, tdir) { return fmt.Errorf("path %s escapes %s, abort", target.RemotePath, tdir) } // check dir if _, err = os.Stat(parentDir); os.IsNotExist(err) { log.Debugf("%s doesn't exist, create", parentDir) if err = os.MkdirAll(parentDir, os.ModePerm); err != nil { return fmt.Errorf("while creating parent directories: %w", err) } } // check actual file if _, err = os.Stat(finalPath); !os.IsNotExist(err) { log.Warningf("%s : overwrite", target.Name) log.Debugf("target: %s/%s", tdir, target.RemotePath) } else { log.Infof("%s : OK", target.Name) } f, err := os.OpenFile(tdir+"/"+target.RemotePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) if err != nil { return fmt.Errorf("while opening file: %w", err) } defer f.Close() _, err = f.Write(body) if err != nil { return fmt.Errorf("while writing file: %w", err) } target.Downloaded = true target.Tainted = false target.UpToDate = true if err = downloadData(h.cfg.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil { return fmt.Errorf("while downloading data for %s: %w", target.FileName, err) } h.Items[target.Type][target.Name] = *target return nil } // DownloadDataIfNeeded downloads the data files for an item func (h *Hub) DownloadDataIfNeeded(target Item, force bool) error { itemFilePath := fmt.Sprintf("%s/%s/%s/%s", h.cfg.InstallDir, target.Type, target.Stage, target.FileName) itemFile, err := os.Open(itemFilePath) if err != nil { return fmt.Errorf("while opening %s: %w", itemFilePath, err) } defer itemFile.Close() if err = downloadData(h.cfg.InstallDataDir, force, itemFile); err != nil { return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err) } return nil } // downloadData downloads the data files for an item func downloadData(dataFolder string, force bool, reader io.Reader) error { var err error dec := yaml.NewDecoder(reader) for { data := &DataSet{} err = dec.Decode(data) if err != nil { if errors.Is(err, io.EOF) { break } return fmt.Errorf("while reading file: %w", err) } download := false for _, dataS := range data.Data { if _, err = os.Stat(filepath.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) { download = true } } if download || force { err = GetData(data.Data, dataFolder) if err != nil { return fmt.Errorf("while getting data: %w", err) } } } return nil }