crowdsec/pkg/types/dataset.go

75 lines
1.3 KiB
Go
Raw Normal View History

2020-05-27 14:31:08 +00:00
package types
import (
"fmt"
"io"
2020-05-27 14:31:08 +00:00
"net/http"
"os"
"path"
log "github.com/sirupsen/logrus"
)
type DataSource struct {
SourceURL string `yaml:"source_url"`
DestPath string `yaml:"dest_file"`
Type string `yaml:"type"`
}
type DataSet struct {
Data []*DataSource `yaml:"data,omitempty"`
}
func downloadFile(url string, destPath string) error {
log.Debugf("downloading %s in %s", url, destPath)
req, err := http.NewRequest(http.MethodGet, url, nil)
2020-05-27 14:31:08 +00:00
if err != nil {
return err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
2020-05-27 14:31:08 +00:00
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
2020-05-27 14:31:08 +00:00
return fmt.Errorf("download response 'HTTP %d' : %s", resp.StatusCode, string(body))
}
2020-05-27 15:57:37 +00:00
file, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
2020-05-27 14:31:08 +00:00
if err != nil {
return err
}
_, err = file.WriteString(string(body))
if err != nil {
return err
}
err = file.Sync()
if err != nil {
return err
}
return nil
}
func GetData(data []*DataSource, dataDir string) error {
for _, dataS := range data {
destPath := path.Join(dataDir, dataS.DestPath)
log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath)
err := downloadFile(dataS.SourceURL, destPath)
if err != nil {
return err
}
}
return nil
}