crowdsec/pkg/cwhub/dataset.go

142 lines
3.2 KiB
Go
Raw Normal View History

package cwhub
2020-05-27 14:31:08 +00:00
import (
"errors"
2020-05-27 14:31:08 +00:00
"fmt"
"io"
2020-05-27 14:31:08 +00:00
"net/http"
"os"
"time"
2020-05-27 14:31:08 +00:00
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v3"
2020-05-27 14:31:08 +00:00
"github.com/crowdsecurity/crowdsec/pkg/types"
)
2020-05-27 14:31:08 +00:00
// The DataSet is a list of data sources required by an item (built from the data: section in the yaml).
2020-05-27 14:31:08 +00:00
type DataSet struct {
Data []types.DataSource `yaml:"data,omitempty"`
2020-05-27 14:31:08 +00:00
}
// downloadFile downloads a file and writes it to disk, with no hash verification.
2020-05-27 14:31:08 +00:00
func downloadFile(url string, destPath string) error {
log.Debugf("downloading %s in %s", url, destPath)
resp, err := hubClient.Get(url)
2020-05-27 14:31:08 +00:00
if err != nil {
return fmt.Errorf("while downloading %s: %w", url, err)
2020-05-27 14:31:08 +00:00
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
2020-05-27 14:31:08 +00:00
}
file, err := os.Create(destPath)
2020-05-27 14:31:08 +00:00
if err != nil {
return err
}
defer file.Close()
2020-05-27 14:31:08 +00:00
// avoid reading the whole file in memory
_, err = io.Copy(file, resp.Body)
2020-05-27 14:31:08 +00:00
if err != nil {
return err
}
if err = file.Sync(); err != nil {
2020-05-27 14:31:08 +00:00
return err
}
return nil
}
// needsUpdate checks if a data file has to be downloaded (or updated).
// if the local file doesn't exist, update.
// if the remote is newer than the local file, update.
// if the remote has no modification date, but local file has been modified > a week ago, update.
func needsUpdate(destPath string, url string) bool {
fileInfo, err := os.Stat(destPath)
switch {
case os.IsNotExist(err):
return true
case err != nil:
log.Errorf("while getting %s: %s", destPath, err)
return true
}
resp, err := hubClient.Head(url)
if err != nil {
log.Errorf("while getting %s: %s", url, err)
// Head failed, Get would likely fail too -> no update
return false
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
log.Errorf("bad http code %d for %s", resp.StatusCode, url)
return false
}
// update if local file is older than this
shelfLife := 7 * 24 * time.Hour
lastModify := fileInfo.ModTime()
localIsOld := lastModify.Add(shelfLife).Before(time.Now())
remoteLastModified := resp.Header.Get("Last-Modified")
if remoteLastModified == "" {
if localIsOld {
log.Infof("no last modified date for %s, but local file is older than %s", url, shelfLife)
}
return localIsOld
}
lastAvailable, err := time.Parse(time.RFC1123, remoteLastModified)
if err != nil {
log.Warningf("while parsing last modified date for %s: %s", url, err)
return localIsOld
}
if lastModify.Before(lastAvailable) {
log.Infof("new version available, updating %s", destPath)
return true
}
return false
}
// downloadDataSet downloads all the data files for an item.
func downloadDataSet(dataFolder string, force bool, reader io.Reader) error {
dec := yaml.NewDecoder(reader)
for {
data := &DataSet{}
if err := dec.Decode(data); err != nil {
if errors.Is(err, io.EOF) {
break
}
return fmt.Errorf("while reading file: %w", err)
}
for _, dataS := range data.Data {
destPath, err := safePath(dataFolder, dataS.DestPath)
if err != nil {
return err
}
if force || needsUpdate(destPath, dataS.SourceURL) {
if err := downloadFile(dataS.SourceURL, destPath); err != nil {
return fmt.Errorf("while getting data: %w", err)
}
}
2020-05-27 14:31:08 +00:00
}
}
return nil
}