pkg/cwhub: download data assets to temporary files to avoid partial fetch (#2879)
This commit is contained in:
parent
1eab943ec2
commit
6c5e8afde9
|
@ -6,6 +6,7 @@ import (
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/sirupsen/logrus"
|
"github.com/sirupsen/logrus"
|
||||||
|
@ -31,19 +32,32 @@ func downloadFile(url string, destPath string) error {
|
||||||
return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
|
return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
|
||||||
}
|
}
|
||||||
|
|
||||||
file, err := os.Create(destPath)
|
tmpFile, err := os.CreateTemp(filepath.Dir(destPath), filepath.Base(destPath)+".*.tmp")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer file.Close()
|
|
||||||
|
tmpFileName := tmpFile.Name()
|
||||||
|
defer func() {
|
||||||
|
tmpFile.Close()
|
||||||
|
os.Remove(tmpFileName)
|
||||||
|
}()
|
||||||
|
|
||||||
// avoid reading the whole file in memory
|
// avoid reading the whole file in memory
|
||||||
_, err = io.Copy(file, resp.Body)
|
_, err = io.Copy(tmpFile, resp.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = file.Sync(); err != nil {
|
if err = tmpFile.Sync(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = tmpFile.Close(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = os.Rename(tmpFileName, destPath); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -6,7 +6,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
// ErrNilRemoteHub is returned when the remote hub configuration is not provided to the NewHub constructor.
|
// ErrNilRemoteHub is returned when trying to download with a local-only configuration.
|
||||||
ErrNilRemoteHub = errors.New("remote hub configuration is not provided. Please report this issue to the developers")
|
ErrNilRemoteHub = errors.New("remote hub configuration is not provided. Please report this issue to the developers")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -3,6 +3,7 @@ package cwhub
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
@ -34,7 +35,7 @@ func (h *Hub) GetDataDir() string {
|
||||||
// All download operations (including updateIndex) return ErrNilRemoteHub if the remote configuration is not set.
|
// All download operations (including updateIndex) return ErrNilRemoteHub if the remote configuration is not set.
|
||||||
func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, updateIndex bool, logger *logrus.Logger) (*Hub, error) {
|
func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, updateIndex bool, logger *logrus.Logger) (*Hub, error) {
|
||||||
if local == nil {
|
if local == nil {
|
||||||
return nil, fmt.Errorf("no hub configuration found")
|
return nil, errors.New("no hub configuration found")
|
||||||
}
|
}
|
||||||
|
|
||||||
if logger == nil {
|
if logger == nil {
|
||||||
|
|
|
@ -93,13 +93,14 @@ func (h *Hub) getItemFileInfo(path string, logger *logrus.Logger) (*itemFileInfo
|
||||||
}
|
}
|
||||||
} else if strings.HasPrefix(path, installDir) { // we're in install /etc/crowdsec/<type>/...
|
} else if strings.HasPrefix(path, installDir) { // we're in install /etc/crowdsec/<type>/...
|
||||||
logger.Tracef("in install dir")
|
logger.Tracef("in install dir")
|
||||||
|
|
||||||
if len(subs) < 3 {
|
if len(subs) < 3 {
|
||||||
return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs))
|
return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs))
|
||||||
}
|
}
|
||||||
///.../config/parser/stage/file.yaml
|
// .../config/parser/stage/file.yaml
|
||||||
///.../config/postoverflow/stage/file.yaml
|
// .../config/postoverflow/stage/file.yaml
|
||||||
///.../config/scenarios/scenar.yaml
|
// .../config/scenarios/scenar.yaml
|
||||||
///.../config/collections/linux.yaml //file is empty
|
// .../config/collections/linux.yaml //file is empty
|
||||||
ret = &itemFileInfo{
|
ret = &itemFileInfo{
|
||||||
inhub: false,
|
inhub: false,
|
||||||
fname: subs[len(subs)-1],
|
fname: subs[len(subs)-1],
|
||||||
|
|
Loading…
Reference in a new issue