2023-06-08 14:49:51 +00:00
|
|
|
package cwhub
|
2020-05-27 14:31:08 +00:00
|
|
|
|
|
|
|
import (
|
2023-10-20 12:32:35 +00:00
|
|
|
"errors"
|
2020-05-27 14:31:08 +00:00
|
|
|
"fmt"
|
2022-09-06 11:55:03 +00:00
|
|
|
"io"
|
2020-05-27 14:31:08 +00:00
|
|
|
"net/http"
|
|
|
|
"os"
|
2023-07-26 08:29:58 +00:00
|
|
|
"path/filepath"
|
2020-05-27 14:31:08 +00:00
|
|
|
|
|
|
|
log "github.com/sirupsen/logrus"
|
2023-10-20 12:32:35 +00:00
|
|
|
"gopkg.in/yaml.v2"
|
2020-05-27 14:31:08 +00:00
|
|
|
|
2023-06-08 14:49:51 +00:00
|
|
|
"github.com/crowdsecurity/crowdsec/pkg/types"
|
|
|
|
)
|
2020-05-27 14:31:08 +00:00
|
|
|
|
|
|
|
type DataSet struct {
|
2023-06-08 14:49:51 +00:00
|
|
|
Data []*types.DataSource `yaml:"data,omitempty"`
|
2020-05-27 14:31:08 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func downloadFile(url string, destPath string) error {
|
|
|
|
log.Debugf("downloading %s in %s", url, destPath)
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2022-08-16 07:46:10 +00:00
|
|
|
req, err := http.NewRequest(http.MethodGet, url, nil)
|
2020-05-27 14:31:08 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
resp, err := http.DefaultClient.Do(req)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer resp.Body.Close()
|
|
|
|
|
2022-09-06 11:55:03 +00:00
|
|
|
body, err := io.ReadAll(resp.Body)
|
2020-05-27 14:31:08 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2022-09-06 11:55:03 +00:00
|
|
|
if resp.StatusCode != http.StatusOK {
|
2020-05-27 14:31:08 +00:00
|
|
|
return fmt.Errorf("download response 'HTTP %d' : %s", resp.StatusCode, string(body))
|
|
|
|
}
|
|
|
|
|
2023-10-20 12:32:35 +00:00
|
|
|
file, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644)
|
2020-05-27 14:31:08 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-08-25 14:22:10 +00:00
|
|
|
_, err = file.Write(body)
|
2020-05-27 14:31:08 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
err = file.Sync()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-06-08 14:49:51 +00:00
|
|
|
func GetData(data []*types.DataSource, dataDir string) error {
|
2020-05-27 14:31:08 +00:00
|
|
|
for _, dataS := range data {
|
2023-07-26 08:29:58 +00:00
|
|
|
destPath := filepath.Join(dataDir, dataS.DestPath)
|
2020-05-27 14:31:08 +00:00
|
|
|
log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath)
|
2023-10-03 09:20:56 +00:00
|
|
|
|
2020-05-27 14:31:08 +00:00
|
|
|
err := downloadFile(dataS.SourceURL, destPath)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
2023-10-20 12:32:35 +00:00
|
|
|
|
|
|
|
// downloadData downloads the data files for an item
|
|
|
|
func downloadData(dataFolder string, force bool, reader io.Reader) error {
|
|
|
|
var err error
|
|
|
|
|
|
|
|
dec := yaml.NewDecoder(reader)
|
|
|
|
|
|
|
|
for {
|
|
|
|
data := &DataSet{}
|
|
|
|
|
|
|
|
err = dec.Decode(data)
|
|
|
|
if err != nil {
|
|
|
|
if errors.Is(err, io.EOF) {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
|
|
|
|
return fmt.Errorf("while reading file: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
download := false
|
|
|
|
|
|
|
|
for _, dataS := range data.Data {
|
|
|
|
if _, err = os.Stat(filepath.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) {
|
|
|
|
download = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if download || force {
|
|
|
|
err = GetData(data.Data, dataFolder)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("while getting data: %w", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|