pkg/cwhub documentation (#2607)
* pkg/cwhub: package documentation * Don't repeat local state in "cscli... inspect" * lint * use proper name of the hub item instead of the filename for local items * hub update: avoid reporting local items as tainted
This commit is contained in:
parent
1509c2d97c
commit
2c652ef92f
|
@ -268,7 +268,7 @@ func NewItemsInstallCmd(typeName string) *cobra.Command {
|
||||||
func istalledParentNames(item *cwhub.Item) []string {
|
func istalledParentNames(item *cwhub.Item) []string {
|
||||||
ret := make([]string, 0)
|
ret := make([]string, 0)
|
||||||
|
|
||||||
for _, parent := range item.ParentCollections() {
|
for _, parent := range item.AncestorCollections() {
|
||||||
if parent.State.Installed {
|
if parent.State.Installed {
|
||||||
ret = append(ret, parent.Name)
|
ret = append(ret, parent.Name)
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,3 @@
|
||||||
// Package cwhub is responsible for installing and upgrading the local hub files.
|
|
||||||
//
|
|
||||||
// This includes retrieving the index, the items to install (parsers, scenarios, data files...)
|
|
||||||
// and managing the dependencies and taints.
|
|
||||||
package cwhub
|
package cwhub
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
|
@ -27,7 +27,7 @@ const mockURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s"
|
||||||
|
|
||||||
var responseByPath map[string]string
|
var responseByPath map[string]string
|
||||||
|
|
||||||
// testHub initializes a temporary hub with an empty json file, optionally updating it
|
// testHub initializes a temporary hub with an empty json file, optionally updating it.
|
||||||
func testHub(t *testing.T, update bool) *Hub {
|
func testHub(t *testing.T, update bool) *Hub {
|
||||||
tmpDir, err := os.MkdirTemp("", "testhub")
|
tmpDir, err := os.MkdirTemp("", "testhub")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
@ -67,7 +67,7 @@ func testHub(t *testing.T, update bool) *Hub {
|
||||||
return hub
|
return hub
|
||||||
}
|
}
|
||||||
|
|
||||||
// envSetup initializes the temporary hub and mocks the http client
|
// envSetup initializes the temporary hub and mocks the http client.
|
||||||
func envSetup(t *testing.T) *Hub {
|
func envSetup(t *testing.T) *Hub {
|
||||||
setResponseByPath()
|
setResponseByPath()
|
||||||
log.SetLevel(log.DebugLevel)
|
log.SetLevel(log.DebugLevel)
|
||||||
|
@ -92,7 +92,7 @@ func newMockTransport() http.RoundTripper {
|
||||||
return &mockTransport{}
|
return &mockTransport{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Implement http.RoundTripper
|
// Implement http.RoundTripper.
|
||||||
func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||||
// Create mocked http.Response
|
// Create mocked http.Response
|
||||||
response := &http.Response{
|
response := &http.Response{
|
||||||
|
|
|
@ -13,11 +13,12 @@ import (
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// The DataSet is a list of data sources required by an item (built from the data: section in the yaml).
|
||||||
type DataSet struct {
|
type DataSet struct {
|
||||||
Data []types.DataSource `yaml:"data,omitempty"`
|
Data []types.DataSource `yaml:"data,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// downloadFile downloads a file and writes it to disk, with no hash verification
|
// downloadFile downloads a file and writes it to disk, with no hash verification.
|
||||||
func downloadFile(url string, destPath string) error {
|
func downloadFile(url string, destPath string) error {
|
||||||
log.Debugf("downloading %s in %s", url, destPath)
|
log.Debugf("downloading %s in %s", url, destPath)
|
||||||
|
|
||||||
|
@ -50,7 +51,7 @@ func downloadFile(url string, destPath string) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// downloadDataSet downloads all the data files for an item
|
// downloadDataSet downloads all the data files for an item.
|
||||||
func downloadDataSet(dataFolder string, force bool, reader io.Reader) error {
|
func downloadDataSet(dataFolder string, force bool, reader io.Reader) error {
|
||||||
dec := yaml.NewDecoder(reader)
|
dec := yaml.NewDecoder(reader)
|
||||||
|
|
||||||
|
|
113
pkg/cwhub/doc.go
Normal file
113
pkg/cwhub/doc.go
Normal file
|
@ -0,0 +1,113 @@
|
||||||
|
// Package cwhub is responsible for installing and upgrading the local hub files for CrowdSec.
|
||||||
|
//
|
||||||
|
// # Definitions
|
||||||
|
//
|
||||||
|
// - A hub ITEM is a file that defines a parser, a scenario, a collection... in the case of a collection, it has dependencies on other hub items.
|
||||||
|
// - The hub INDEX is a JSON file that contains a tree of available hub items.
|
||||||
|
// - A REMOTE HUB is an HTTP server that hosts the hub index and the hub items. It can serve from several branches, usually linked to the CrowdSec version.
|
||||||
|
// - A LOCAL HUB is a directory that contains a copy of the hub index and the downloaded hub items.
|
||||||
|
//
|
||||||
|
// Once downloaded, hub items can be installed by linking to them from the configuration directory.
|
||||||
|
// If an item is present in the configuration directory but it's not a link to the local hub, it is
|
||||||
|
// considered as a LOCAL ITEM and won't be removed or upgraded.
|
||||||
|
//
|
||||||
|
// # Directory Structure
|
||||||
|
//
|
||||||
|
// A typical directory layout is the following:
|
||||||
|
//
|
||||||
|
// For the local hub (HubDir = /etc/crowdsec/hub):
|
||||||
|
//
|
||||||
|
// - /etc/crowdsec/hub/.index.json
|
||||||
|
// - /etc/crowdsec/hub/parsers/{stage}/{author}/{parser-name}.yaml
|
||||||
|
// - /etc/crowdsec/hub/scenarios/{author}/{scenario-name}.yaml
|
||||||
|
//
|
||||||
|
// For the configuration directory (InstallDir = /etc/crowdsec):
|
||||||
|
//
|
||||||
|
// - /etc/crowdsec/parsers/{stage}/{parser-name.yaml} -> /etc/crowdsec/hub/parsers/{stage}/{author}/{parser-name}.yaml
|
||||||
|
// - /etc/crowdsec/scenarios/{scenario-name.yaml} -> /etc/crowdsec/hub/scenarios/{author}/{scenario-name}.yaml
|
||||||
|
// - /etc/crowdsec/scenarios/local-scenario.yaml
|
||||||
|
//
|
||||||
|
// Note that installed items are not grouped by author, this may change in the future if we want to
|
||||||
|
// support items with the same name from different authors.
|
||||||
|
//
|
||||||
|
// Only parsers and postoverflows have the concept of stage.
|
||||||
|
//
|
||||||
|
// Additionally, an item can reference a DATA SET that is installed in a different location than
|
||||||
|
// the item itself. These files are stored in the data directory (InstallDataDir = /var/lib/crowdsec/data).
|
||||||
|
//
|
||||||
|
// - /var/lib/crowdsec/data/http_path_traversal.txt
|
||||||
|
// - /var/lib/crowdsec/data/jira_cve_2021-26086.txt
|
||||||
|
// - /var/lib/crowdsec/data/log4j2_cve_2021_44228.txt
|
||||||
|
// - /var/lib/crowdsec/data/sensitive_data.txt
|
||||||
|
//
|
||||||
|
//
|
||||||
|
// # Using the package
|
||||||
|
//
|
||||||
|
// The main entry point is the Hub struct. You can create a new instance with NewHub().
|
||||||
|
// This constructor takes three parameters, but only the LOCAL HUB configuration is required:
|
||||||
|
//
|
||||||
|
// import (
|
||||||
|
// "fmt"
|
||||||
|
// "github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||||
|
// "github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||||
|
// )
|
||||||
|
//
|
||||||
|
// localHub := csconfig.LocalHubCfg{
|
||||||
|
// HubIndexFile: "/etc/crowdsec/hub/.index.json",
|
||||||
|
// HubDir: "/etc/crowdsec/hub",
|
||||||
|
// InstallDir: "/etc/crowdsec",
|
||||||
|
// InstallDataDir: "/var/lib/crowdsec/data",
|
||||||
|
// }
|
||||||
|
// hub, err := cwhub.NewHub(localHub, nil, false)
|
||||||
|
// if err != nil {
|
||||||
|
// return fmt.Errorf("unable to initialize hub: %w", err)
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// Now you can use the hub to access the existing items:
|
||||||
|
//
|
||||||
|
// // list all the parsers
|
||||||
|
// for _, parser := range hub.GetItemMap(cwhub.PARSERS) {
|
||||||
|
// fmt.Printf("parser: %s\n", parser.Name)
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// // retrieve a specific collection
|
||||||
|
// coll := hub.GetItem(cwhub.COLLECTIONS, "crowdsecurity/linux")
|
||||||
|
// if coll == nil {
|
||||||
|
// return fmt.Errorf("collection not found")
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// You can also install items if they have already been downloaded:
|
||||||
|
//
|
||||||
|
// // install a parser
|
||||||
|
// force := false
|
||||||
|
// downloadOnly := false
|
||||||
|
// err := parser.Install(force, downloadOnly)
|
||||||
|
// if err != nil {
|
||||||
|
// return fmt.Errorf("unable to install parser: %w", err)
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// As soon as you try to install an item that is not downloaded or is not up-to-date (meaning its computed hash
|
||||||
|
// does not correspond to the latest version available in the index), a download will be attempted and you'll
|
||||||
|
// get the error "remote hub configuration is not provided".
|
||||||
|
//
|
||||||
|
// To provide the remote hub configuration, use the second parameter of NewHub():
|
||||||
|
//
|
||||||
|
// remoteHub := cwhub.RemoteHubCfg{
|
||||||
|
// URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s",
|
||||||
|
// Branch: "master",
|
||||||
|
// IndexPath: ".index.json",
|
||||||
|
// }
|
||||||
|
// updateIndex := false
|
||||||
|
// hub, err := cwhub.NewHub(localHub, remoteHub, updateIndex)
|
||||||
|
// if err != nil {
|
||||||
|
// return fmt.Errorf("unable to initialize hub: %w", err)
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// The URLTemplate is a string that will be used to build the URL of the remote hub. It must contain two
|
||||||
|
// placeholders: the branch and the file path (it will be an index or an item).
|
||||||
|
//
|
||||||
|
// Setting the third parameter to true will download the latest version of the index, if available on the
|
||||||
|
// specified branch.
|
||||||
|
// There is no exported method to update the index once the hub struct is created.
|
||||||
|
//
|
||||||
|
package cwhub
|
|
@ -11,8 +11,8 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
// installPath returns the location of the symlink to the item in the hub, or the path of the item itself if it's local
|
// installPath returns the location of the symlink to the item in the hub, or the path of the item itself if it's local
|
||||||
// (eg. /etc/crowdsec/collections/xyz.yaml)
|
// (eg. /etc/crowdsec/collections/xyz.yaml).
|
||||||
// raises an error if the path goes outside of the install dir
|
// Raises an error if the path goes outside of the install dir.
|
||||||
func (i *Item) installPath() (string, error) {
|
func (i *Item) installPath() (string, error) {
|
||||||
p := i.Type
|
p := i.Type
|
||||||
if i.Stage != "" {
|
if i.Stage != "" {
|
||||||
|
@ -23,8 +23,8 @@ func (i *Item) installPath() (string, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// downloadPath returns the location of the actual config file in the hub
|
// downloadPath returns the location of the actual config file in the hub
|
||||||
// (eg. /etc/crowdsec/hub/collections/author/xyz.yaml)
|
// (eg. /etc/crowdsec/hub/collections/author/xyz.yaml).
|
||||||
// raises an error if the path goes outside of the hub dir
|
// Raises an error if the path goes outside of the hub dir.
|
||||||
func (i *Item) downloadPath() (string, error) {
|
func (i *Item) downloadPath() (string, error) {
|
||||||
ret, err := safePath(i.hub.local.HubDir, i.RemotePath)
|
ret, err := safePath(i.hub.local.HubDir, i.RemotePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -34,7 +34,7 @@ func (i *Item) downloadPath() (string, error) {
|
||||||
return ret, nil
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// makeLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir
|
// makeLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir.
|
||||||
func (i *Item) createInstallLink() error {
|
func (i *Item) createInstallLink() error {
|
||||||
dest, err := i.installPath()
|
dest, err := i.installPath()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -63,7 +63,7 @@ func (i *Item) createInstallLink() error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// enable enables the item by creating a symlink to the downloaded content, and also enables sub-items
|
// enable enables the item by creating a symlink to the downloaded content, and also enables sub-items.
|
||||||
func (i *Item) enable() error {
|
func (i *Item) enable() error {
|
||||||
if i.State.Installed {
|
if i.State.Installed {
|
||||||
if i.State.Tainted {
|
if i.State.Tainted {
|
||||||
|
@ -97,7 +97,7 @@ func (i *Item) enable() error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// purge removes the actual config file that was downloaded
|
// purge removes the actual config file that was downloaded.
|
||||||
func (i *Item) purge() error {
|
func (i *Item) purge() error {
|
||||||
if !i.State.Downloaded {
|
if !i.State.Downloaded {
|
||||||
log.Infof("removing %s: not downloaded -- no need to remove", i.Name)
|
log.Infof("removing %s: not downloaded -- no need to remove", i.Name)
|
||||||
|
@ -124,7 +124,7 @@ func (i *Item) purge() error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// removeInstallLink removes the symlink to the downloaded content
|
// removeInstallLink removes the symlink to the downloaded content.
|
||||||
func (i *Item) removeInstallLink() error {
|
func (i *Item) removeInstallLink() error {
|
||||||
syml, err := i.installPath()
|
syml, err := i.installPath()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -166,7 +166,7 @@ func (i *Item) removeInstallLink() error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// disable removes the install link, and optionally the downloaded content
|
// disable removes the install link, and optionally the downloaded content.
|
||||||
func (i *Item) disable(purge bool, force bool) error {
|
func (i *Item) disable(purge bool, force bool) error {
|
||||||
// XXX: should return the number of disabled/purged items to inform the upper layer whether to reload or not
|
// XXX: should return the number of disabled/purged items to inform the upper layer whether to reload or not
|
||||||
err := i.removeInstallLink()
|
err := i.removeInstallLink()
|
||||||
|
|
|
@ -7,10 +7,10 @@ import (
|
||||||
|
|
||||||
var (
|
var (
|
||||||
// ErrNilRemoteHub is returned when the remote hub configuration is not provided to the NewHub constructor.
|
// ErrNilRemoteHub is returned when the remote hub configuration is not provided to the NewHub constructor.
|
||||||
// All attempts to download index or items will return this error.
|
|
||||||
ErrNilRemoteHub = errors.New("remote hub configuration is not provided. Please report this issue to the developers")
|
ErrNilRemoteHub = errors.New("remote hub configuration is not provided. Please report this issue to the developers")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// IndexNotFoundError is returned when the remote hub index is not found.
|
||||||
type IndexNotFoundError struct {
|
type IndexNotFoundError struct {
|
||||||
URL string
|
URL string
|
||||||
Branch string
|
Branch string
|
||||||
|
|
|
@ -19,7 +19,7 @@ import (
|
||||||
"slices"
|
"slices"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Install installs the item from the hub, downloading it if needed
|
// Install installs the item from the hub, downloading it if needed.
|
||||||
func (i *Item) Install(force bool, downloadOnly bool) error {
|
func (i *Item) Install(force bool, downloadOnly bool) error {
|
||||||
if downloadOnly && i.State.Downloaded && i.State.UpToDate {
|
if downloadOnly && i.State.Downloaded && i.State.UpToDate {
|
||||||
log.Infof("%s is already downloaded and up-to-date", i.Name)
|
log.Infof("%s is already downloaded and up-to-date", i.Name)
|
||||||
|
@ -52,7 +52,7 @@ func (i *Item) Install(force bool, downloadOnly bool) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// allDependencies returns a list of all (direct or indirect) dependencies of the item
|
// allDependencies returns a list of all (direct or indirect) dependencies of the item.
|
||||||
func (i *Item) allDependencies() ([]*Item, error) {
|
func (i *Item) allDependencies() ([]*Item, error) {
|
||||||
var collectSubItems func(item *Item, visited map[*Item]bool, result *[]*Item) error
|
var collectSubItems func(item *Item, visited map[*Item]bool, result *[]*Item) error
|
||||||
|
|
||||||
|
@ -94,7 +94,7 @@ func (i *Item) allDependencies() ([]*Item, error) {
|
||||||
return ret, nil
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove disables the item, optionally removing the downloaded content
|
// Remove disables the item, optionally removing the downloaded content.
|
||||||
func (i *Item) Remove(purge bool, force bool) (bool, error) {
|
func (i *Item) Remove(purge bool, force bool) (bool, error) {
|
||||||
if i.IsLocal() {
|
if i.IsLocal() {
|
||||||
return false, fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name)
|
return false, fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name)
|
||||||
|
@ -123,7 +123,7 @@ func (i *Item) Remove(purge bool, force bool) (bool, error) {
|
||||||
|
|
||||||
// if the sub depends on a collection that is not a direct or indirect dependency
|
// if the sub depends on a collection that is not a direct or indirect dependency
|
||||||
// of the current item, it is not removed
|
// of the current item, it is not removed
|
||||||
for _, subParent := range sub.ParentCollections() {
|
for _, subParent := range sub.AncestorCollections() {
|
||||||
if !purge && !subParent.State.Installed {
|
if !purge && !subParent.State.Installed {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
@ -156,7 +156,7 @@ func (i *Item) Remove(purge bool, force bool) (bool, error) {
|
||||||
return removed, nil
|
return removed, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Upgrade downloads and applies the last version from the hub
|
// Upgrade downloads and applies the last version of the item from the hub.
|
||||||
func (i *Item) Upgrade(force bool) (bool, error) {
|
func (i *Item) Upgrade(force bool) (bool, error) {
|
||||||
updated := false
|
updated := false
|
||||||
|
|
||||||
|
@ -203,7 +203,7 @@ func (i *Item) Upgrade(force bool) (bool, error) {
|
||||||
return updated, nil
|
return updated, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// downloadLatest downloads the latest version of the item to the hub directory
|
// downloadLatest downloads the latest version of the item to the hub directory.
|
||||||
func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) {
|
func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) {
|
||||||
// XXX: should return the path of the downloaded file (taken from download())
|
// XXX: should return the path of the downloaded file (taken from download())
|
||||||
log.Debugf("Downloading %s %s", i.Type, i.Name)
|
log.Debugf("Downloading %s %s", i.Type, i.Name)
|
||||||
|
@ -253,7 +253,7 @@ func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) {
|
||||||
return ret, nil
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// fetch downloads the item from the hub, verifies the hash and returns the content
|
// fetch downloads the item from the hub, verifies the hash and returns the content.
|
||||||
func (i *Item) fetch() ([]byte, error) {
|
func (i *Item) fetch() ([]byte, error) {
|
||||||
url, err := i.hub.remote.urlTo(i.RemotePath)
|
url, err := i.hub.remote.urlTo(i.RemotePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -291,7 +291,7 @@ func (i *Item) fetch() ([]byte, error) {
|
||||||
return body, nil
|
return body, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// download downloads the item from the hub and writes it to the hub directory
|
// download downloads the item from the hub and writes it to the hub directory.
|
||||||
func (i *Item) download(overwrite bool) (string, error) {
|
func (i *Item) download(overwrite bool) (string, error) {
|
||||||
// if user didn't --force, don't overwrite local, tainted, up-to-date files
|
// if user didn't --force, don't overwrite local, tainted, up-to-date files
|
||||||
if !overwrite {
|
if !overwrite {
|
||||||
|
@ -348,7 +348,7 @@ func (i *Item) download(overwrite bool) (string, error) {
|
||||||
return finalPath, nil
|
return finalPath, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// DownloadDataIfNeeded downloads the data files for the item
|
// DownloadDataIfNeeded downloads the data set for the item.
|
||||||
func (i *Item) DownloadDataIfNeeded(force bool) error {
|
func (i *Item) DownloadDataIfNeeded(force bool) error {
|
||||||
itemFilePath, err := i.installPath()
|
itemFilePath, err := i.installPath()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -8,8 +8,8 @@ import (
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection
|
// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection.
|
||||||
// We expect the new scenario to be installed
|
// We expect the new scenario to be installed.
|
||||||
func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
|
func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
|
||||||
hub := envSetup(t)
|
hub := envSetup(t)
|
||||||
|
|
||||||
|
@ -110,7 +110,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
|
||||||
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
|
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
|
||||||
}
|
}
|
||||||
|
|
||||||
// getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test
|
// getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test.
|
||||||
func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCfg) *Hub {
|
func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCfg) *Hub {
|
||||||
hub, err := NewHub(local, remote, false)
|
hub, err := NewHub(local, remote, false)
|
||||||
require.NoError(t, err, "failed to load hub index")
|
require.NoError(t, err, "failed to load hub index")
|
||||||
|
|
|
@ -13,19 +13,22 @@ import (
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Hub is the main structure for the package.
|
||||||
type Hub struct {
|
type Hub struct {
|
||||||
Items HubItems
|
Items HubItems // Items read from HubDir and InstallDir
|
||||||
local *csconfig.LocalHubCfg
|
local *csconfig.LocalHubCfg
|
||||||
remote *RemoteHubCfg
|
remote *RemoteHubCfg
|
||||||
Warnings []string
|
Warnings []string // Warnings encountered during sync
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetDataDir returns the data directory, where data sets are installed.
|
||||||
func (h *Hub) GetDataDir() string {
|
func (h *Hub) GetDataDir() string {
|
||||||
return h.local.InstallDataDir
|
return h.local.InstallDataDir
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewHub returns a new Hub instance with local and (optionally) remote configuration, and syncs the local state
|
// NewHub returns a new Hub instance with local and (optionally) remote configuration, and syncs the local state.
|
||||||
// It also downloads the index if updateIndex is true
|
// If updateIndex is true, the local index file is updated from the remote before reading the state of the items.
|
||||||
|
// All download operations (including updateIndex) return ErrNilRemoteHub if the remote configuration is not set.
|
||||||
func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, updateIndex bool) (*Hub, error) {
|
func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, updateIndex bool) (*Hub, error) {
|
||||||
if local == nil {
|
if local == nil {
|
||||||
return nil, fmt.Errorf("no hub configuration found")
|
return nil, fmt.Errorf("no hub configuration found")
|
||||||
|
@ -55,7 +58,7 @@ func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, updateIndex bool)
|
||||||
return hub, nil
|
return hub, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseIndex takes the content of an index file and fills the map of associated parsers/scenarios/collections
|
// parseIndex takes the content of an index file and fills the map of associated parsers/scenarios/collections.
|
||||||
func (h *Hub) parseIndex() error {
|
func (h *Hub) parseIndex() error {
|
||||||
bidx, err := os.ReadFile(h.local.HubIndexFile)
|
bidx, err := os.ReadFile(h.local.HubIndexFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -91,7 +94,7 @@ func (h *Hub) parseIndex() error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ItemStats returns total counts of the hub items
|
// ItemStats returns total counts of the hub items, including local and tainted.
|
||||||
func (h *Hub) ItemStats() []string {
|
func (h *Hub) ItemStats() []string {
|
||||||
loaded := ""
|
loaded := ""
|
||||||
local := 0
|
local := 0
|
||||||
|
@ -131,7 +134,7 @@ func (h *Hub) ItemStats() []string {
|
||||||
return ret
|
return ret
|
||||||
}
|
}
|
||||||
|
|
||||||
// updateIndex downloads the latest version of the index and writes it to disk if it changed
|
// updateIndex downloads the latest version of the index and writes it to disk if it changed.
|
||||||
func (h *Hub) updateIndex() error {
|
func (h *Hub) updateIndex() error {
|
||||||
body, err := h.remote.fetchIndex()
|
body, err := h.remote.fetchIndex()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -12,7 +12,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// managed item types
|
// managed item types.
|
||||||
COLLECTIONS = "collections"
|
COLLECTIONS = "collections"
|
||||||
PARSERS = "parsers"
|
PARSERS = "parsers"
|
||||||
POSTOVERFLOWS = "postoverflows"
|
POSTOVERFLOWS = "postoverflows"
|
||||||
|
@ -20,59 +20,57 @@ const (
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
VersionUpToDate = iota
|
versionUpToDate = iota // the latest version from index is installed
|
||||||
VersionUpdateAvailable
|
versionUpdateAvailable // not installed, or lower than latest
|
||||||
VersionUnknown
|
versionUnknown // local file with no version, or invalid version number
|
||||||
VersionFuture
|
versionFuture // local version is higher latest, but is included in the index: should not happen
|
||||||
)
|
)
|
||||||
|
|
||||||
// The order is important, as it is used to range over sub-items in collections
|
var (
|
||||||
var ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS}
|
// The order is important, as it is used to range over sub-items in collections.
|
||||||
|
ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS}
|
||||||
|
)
|
||||||
|
|
||||||
type HubItems map[string]map[string]*Item
|
type HubItems map[string]map[string]*Item
|
||||||
|
|
||||||
// ItemVersion is used to detect the version of a given item
|
// ItemVersion is used to detect the version of a given item
|
||||||
// by comparing the hash of each version to the local file.
|
// by comparing the hash of each version to the local file.
|
||||||
// If the item does not match any known version, it is considered tainted.
|
// If the item does not match any known version, it is considered tainted (modified).
|
||||||
type ItemVersion struct {
|
type ItemVersion struct {
|
||||||
Digest string `json:"digest,omitempty"` // meow
|
Digest string `json:"digest,omitempty"` // meow
|
||||||
Deprecated bool `json:"deprecated,omitempty"`
|
Deprecated bool `json:"deprecated,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ItemState is used to keep the local state (i.e. at runtime) of an item
|
// ItemState is used to keep the local state (i.e. at runtime) of an item.
|
||||||
// This data is not stored in the index, but is displayed in the output of "cscli ... inspect"
|
// This data is not stored in the index, but is displayed with "cscli ... inspect".
|
||||||
type ItemState struct {
|
type ItemState struct {
|
||||||
LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR}
|
LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"`
|
||||||
LocalVersion string `json:"local_version,omitempty"`
|
LocalVersion string `json:"local_version,omitempty"`
|
||||||
LocalHash string `json:"local_hash,omitempty"` // the local meow
|
LocalHash string `json:"local_hash,omitempty"`
|
||||||
Installed bool `json:"installed"`
|
Installed bool `json:"installed"`
|
||||||
Downloaded bool `json:"downloaded"`
|
Downloaded bool `json:"downloaded"`
|
||||||
UpToDate bool `json:"up_to_date"`
|
UpToDate bool `json:"up_to_date"`
|
||||||
Tainted bool `json:"tainted"` // has it been locally modified?
|
Tainted bool `json:"tainted"`
|
||||||
BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any
|
BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Item represents an object managed in the hub. It can be a parser, scenario, collection..
|
// Item is created from an index file and enriched with local info.
|
||||||
type Item struct {
|
type Item struct {
|
||||||
// back pointer to the hub, to retrieve subitems and call install/remove methods
|
hub *Hub // back pointer to the hub, to retrieve other items and call install/remove methods
|
||||||
hub *Hub
|
|
||||||
|
|
||||||
// local (deployed) info
|
State ItemState `json:"-" yaml:"-"` // local state, not stored in the index
|
||||||
State ItemState
|
|
||||||
|
|
||||||
// descriptive info
|
Type string `json:"type,omitempty" yaml:"type,omitempty"` // one of the ItemTypes
|
||||||
Type string `json:"type,omitempty" yaml:"type,omitempty"` // can be any of the ItemTypes
|
|
||||||
Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-...
|
Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-...
|
||||||
Name string `json:"name,omitempty"` // as seen in .index.json, usually "author/name"
|
Name string `json:"name,omitempty"` // usually "author/name"
|
||||||
FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml
|
FileName string `json:"file_name,omitempty"` // eg. apache2-logs.yaml
|
||||||
Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .index.json
|
Description string `json:"description,omitempty" yaml:"description,omitempty"`
|
||||||
Author string `json:"author,omitempty"` // as seen in .index.json
|
Author string `json:"author,omitempty"`
|
||||||
References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .index.json
|
References []string `json:"references,omitempty" yaml:"references,omitempty"`
|
||||||
|
|
||||||
// remote (hub) info
|
RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // path relative to the base URL eg. /parsers/stage/author/file.yaml
|
||||||
RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // the path relative to (git | hub API) ie. /parsers/stage/author/file.yaml
|
Version string `json:"version,omitempty"` // the last available version
|
||||||
Version string `json:"version,omitempty"` // the last version
|
Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // all the known versions
|
||||||
Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // the list of existing versions
|
|
||||||
|
|
||||||
// if it's a collection, it can have sub items
|
// if it's a collection, it can have sub items
|
||||||
Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"`
|
Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"`
|
||||||
|
@ -81,17 +79,18 @@ type Item struct {
|
||||||
Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"`
|
Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// HasSubItems returns true if items of this type can have sub-items. Currently only collections.
|
||||||
func (i *Item) HasSubItems() bool {
|
func (i *Item) HasSubItems() bool {
|
||||||
return i.Type == COLLECTIONS
|
return i.Type == COLLECTIONS
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IsLocal returns true if the item has been create by a user (not downloaded from the hub).
|
||||||
func (i *Item) IsLocal() bool {
|
func (i *Item) IsLocal() bool {
|
||||||
return i.State.Installed && !i.State.Downloaded
|
return i.State.Installed && !i.State.Downloaded
|
||||||
}
|
}
|
||||||
|
|
||||||
// MarshalJSON is used to add the "local" field to the json output
|
// MarshalJSON is used to prepare the output for "cscli ... inspect -o json".
|
||||||
// (i.e. with cscli ... inspect -o json)
|
// It must not use a pointer receiver.
|
||||||
// It must not use a pointer receiver
|
|
||||||
func (i Item) MarshalJSON() ([]byte, error) {
|
func (i Item) MarshalJSON() ([]byte, error) {
|
||||||
type Alias Item
|
type Alias Item
|
||||||
|
|
||||||
|
@ -121,9 +120,8 @@ func (i Item) MarshalJSON() ([]byte, error) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// MarshalYAML is used to add the "local" field to the yaml output
|
// MarshalYAML is used to prepare the output for "cscli ... inspect -o raw".
|
||||||
// (i.e. with cscli ... inspect -o raw)
|
// It must not use a pointer receiver.
|
||||||
// It must not use a pointer receiver
|
|
||||||
func (i Item) MarshalYAML() (interface{}, error) {
|
func (i Item) MarshalYAML() (interface{}, error) {
|
||||||
type Alias Item
|
type Alias Item
|
||||||
|
|
||||||
|
@ -138,7 +136,7 @@ func (i Item) MarshalYAML() (interface{}, error) {
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// SubItems returns a slice of sub-item pointers, excluding the ones that were not found
|
// SubItems returns a slice of sub-items, excluding the ones that were not found.
|
||||||
func (i *Item) SubItems() []*Item {
|
func (i *Item) SubItems() []*Item {
|
||||||
sub := make([]*Item, 0)
|
sub := make([]*Item, 0)
|
||||||
|
|
||||||
|
@ -211,8 +209,8 @@ func (i *Item) logMissingSubItems() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParentCollections returns the list of items (collections) that have this item as a direct dependency
|
// AncestorCollections returns a slice of items (collections) that have this item as a direct or indirect dependency.
|
||||||
func (i *Item) ParentCollections() []*Item {
|
func (i *Item) AncestorCollections() []*Item {
|
||||||
ret := make([]*Item, 0)
|
ret := make([]*Item, 0)
|
||||||
|
|
||||||
for _, parentName := range i.State.BelongsToCollections {
|
for _, parentName := range i.State.BelongsToCollections {
|
||||||
|
@ -228,7 +226,7 @@ func (i *Item) ParentCollections() []*Item {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Status returns the status of the item as a string and an emoji
|
// Status returns the status of the item as a string and an emoji
|
||||||
// ie. "enabled,update-available" and emoji.Warning
|
// (eg. "enabled,update-available" and emoji.Warning).
|
||||||
func (i *Item) Status() (string, emoji.Emoji) {
|
func (i *Item) Status() (string, emoji.Emoji) {
|
||||||
status := "disabled"
|
status := "disabled"
|
||||||
ok := false
|
ok := false
|
||||||
|
@ -269,47 +267,47 @@ func (i *Item) Status() (string, emoji.Emoji) {
|
||||||
return status, emo
|
return status, emo
|
||||||
}
|
}
|
||||||
|
|
||||||
// versionStatus: semver requires 'v' prefix
|
// versionStatus returns the status of the item version compared to the hub version.
|
||||||
|
// semver requires the 'v' prefix.
|
||||||
func (i *Item) versionStatus() int {
|
func (i *Item) versionStatus() int {
|
||||||
local, err := semver.NewVersion(i.State.LocalVersion)
|
local, err := semver.NewVersion(i.State.LocalVersion)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return VersionUnknown
|
return versionUnknown
|
||||||
}
|
}
|
||||||
|
|
||||||
// hub versions are already validated while syncing, ignore errors
|
// hub versions are already validated while syncing, ignore errors
|
||||||
latest, _ := semver.NewVersion(i.Version)
|
latest, _ := semver.NewVersion(i.Version)
|
||||||
|
|
||||||
if local.LessThan(latest) {
|
if local.LessThan(latest) {
|
||||||
return VersionUpdateAvailable
|
return versionUpdateAvailable
|
||||||
}
|
}
|
||||||
|
|
||||||
if local.Equal(latest) {
|
if local.Equal(latest) {
|
||||||
return VersionUpToDate
|
return versionUpToDate
|
||||||
}
|
}
|
||||||
|
|
||||||
return VersionFuture
|
return versionFuture
|
||||||
}
|
}
|
||||||
|
|
||||||
// validPath returns true if the (relative) path is allowed for the item
|
// validPath returns true if the (relative) path is allowed for the item.
|
||||||
// dirNname: the directory name (ie. crowdsecurity)
|
// dirNname: the directory name (ie. crowdsecurity).
|
||||||
// fileName: the filename (ie. apache2-logs.yaml)
|
// fileName: the filename (ie. apache2-logs.yaml).
|
||||||
func (i *Item) validPath(dirName, fileName string) bool {
|
func (i *Item) validPath(dirName, fileName string) bool {
|
||||||
return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml")
|
return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml")
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetItemMap returns the map of items for a given type
|
// GetItemMap returns the map of items for a given type.
|
||||||
func (h *Hub) GetItemMap(itemType string) map[string]*Item {
|
func (h *Hub) GetItemMap(itemType string) map[string]*Item {
|
||||||
return h.Items[itemType]
|
return h.Items[itemType]
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetItem returns the item from hub based on its type and full name (author/name)
|
// GetItem returns an item from hub based on its type and full name (author/name).
|
||||||
func (h *Hub) GetItem(itemType string, itemName string) *Item {
|
func (h *Hub) GetItem(itemType string, itemName string) *Item {
|
||||||
return h.GetItemMap(itemType)[itemName]
|
return h.GetItemMap(itemType)[itemName]
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetItemNames returns the list of (full) item names for a given type
|
// GetItemNames returns a slice of (full) item names for a given type
|
||||||
// ie. for collections: crowdsecurity/apache2 crowdsecurity/nginx
|
// (eg. for collections: crowdsecurity/apache2 crowdsecurity/nginx).
|
||||||
// The names can be used to retrieve the item with GetItem()
|
|
||||||
func (h *Hub) GetItemNames(itemType string) []string {
|
func (h *Hub) GetItemNames(itemType string) []string {
|
||||||
m := h.GetItemMap(itemType)
|
m := h.GetItemMap(itemType)
|
||||||
if m == nil {
|
if m == nil {
|
||||||
|
@ -324,7 +322,7 @@ func (h *Hub) GetItemNames(itemType string) []string {
|
||||||
return names
|
return names
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetAllItems returns a slice of all the items, installed or not
|
// GetAllItems returns a slice of all the items of a given type, installed or not.
|
||||||
func (h *Hub) GetAllItems(itemType string) ([]*Item, error) {
|
func (h *Hub) GetAllItems(itemType string) ([]*Item, error) {
|
||||||
items, ok := h.Items[itemType]
|
items, ok := h.Items[itemType]
|
||||||
if !ok {
|
if !ok {
|
||||||
|
@ -343,7 +341,7 @@ func (h *Hub) GetAllItems(itemType string) ([]*Item, error) {
|
||||||
return ret, nil
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetInstalledItems returns the list of installed items
|
// GetInstalledItems returns a slice of the installed items of a given type.
|
||||||
func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) {
|
func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) {
|
||||||
items, ok := h.Items[itemType]
|
items, ok := h.Items[itemType]
|
||||||
if !ok {
|
if !ok {
|
||||||
|
@ -361,7 +359,7 @@ func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) {
|
||||||
return retItems, nil
|
return retItems, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetInstalledItemNames returns the names of the installed items
|
// GetInstalledItemNames returns the names of the installed items of a given type.
|
||||||
func (h *Hub) GetInstalledItemNames(itemType string) ([]string, error) {
|
func (h *Hub) GetInstalledItemNames(itemType string) ([]string, error) {
|
||||||
items, err := h.GetInstalledItems(itemType)
|
items, err := h.GetInstalledItems(itemType)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -377,7 +375,7 @@ func (h *Hub) GetInstalledItemNames(itemType string) ([]string, error) {
|
||||||
return retStr, nil
|
return retStr, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// SortItemSlice sorts a slice of items by name, case insensitive
|
// SortItemSlice sorts a slice of items by name, case insensitive.
|
||||||
func SortItemSlice(items []*Item) {
|
func SortItemSlice(items []*Item) {
|
||||||
sort.Slice(items, func(i, j int) bool {
|
sort.Slice(items, func(i, j int) bool {
|
||||||
return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name)
|
return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name)
|
||||||
|
|
|
@ -10,7 +10,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary
|
// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary.
|
||||||
func itemKey(itemPath string) (string, error) {
|
func itemKey(itemPath string) (string, error) {
|
||||||
f, err := os.Lstat(itemPath)
|
f, err := os.Lstat(itemPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -37,7 +37,7 @@ func itemKey(itemPath string) (string, error) {
|
||||||
return fmt.Sprintf("%s/%s", author, fname), nil
|
return fmt.Sprintf("%s/%s", author, fname), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetItemByPath retrieves the item from the hub index based on its path.
|
// GetItemByPath retrieves an item from the hub index based on its local path.
|
||||||
func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) {
|
func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) {
|
||||||
itemKey, err := itemKey(itemPath)
|
itemKey, err := itemKey(itemPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -6,14 +6,14 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
)
|
)
|
||||||
|
|
||||||
// RemoteHubCfg contains where to find the remote hub, which branch etc.
|
// RemoteHubCfg is used to retrieve index and items from the remote hub.
|
||||||
type RemoteHubCfg struct {
|
type RemoteHubCfg struct {
|
||||||
Branch string
|
Branch string
|
||||||
URLTemplate string
|
URLTemplate string
|
||||||
IndexPath string
|
IndexPath string
|
||||||
}
|
}
|
||||||
|
|
||||||
// urlTo builds the URL to download a file from the remote hub
|
// urlTo builds the URL to download a file from the remote hub.
|
||||||
func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) {
|
func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) {
|
||||||
if r == nil {
|
if r == nil {
|
||||||
return "", ErrNilRemoteHub
|
return "", ErrNilRemoteHub
|
||||||
|
@ -27,7 +27,7 @@ func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) {
|
||||||
return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil
|
return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// fetchIndex downloads the index from the hub and returns the content
|
// fetchIndex downloads the index from the hub and returns the content.
|
||||||
func (r *RemoteHubCfg) fetchIndex() ([]byte, error) {
|
func (r *RemoteHubCfg) fetchIndex() ([]byte, error) {
|
||||||
if r == nil {
|
if r == nil {
|
||||||
return nil, ErrNilRemoteHub
|
return nil, ErrNilRemoteHub
|
||||||
|
|
|
@ -12,13 +12,14 @@ import (
|
||||||
|
|
||||||
"github.com/Masterminds/semver/v3"
|
"github.com/Masterminds/semver/v3"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
func isYAMLFileName(path string) bool {
|
func isYAMLFileName(path string) bool {
|
||||||
return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")
|
return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")
|
||||||
}
|
}
|
||||||
|
|
||||||
// linkTarget returns the target of a symlink, or empty string if it's dangling
|
// linkTarget returns the target of a symlink, or empty string if it's dangling.
|
||||||
func linkTarget(path string) (string, error) {
|
func linkTarget(path string) (string, error) {
|
||||||
hubpath, err := os.Readlink(path)
|
hubpath, err := os.Readlink(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -52,7 +53,7 @@ func getSHA256(filepath string) (string, error) {
|
||||||
return hex.EncodeToString(h.Sum(nil)), nil
|
return hex.EncodeToString(h.Sum(nil)), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// information used to create a new Item, from a file path
|
// information used to create a new Item, from a file path.
|
||||||
type itemFileInfo struct {
|
type itemFileInfo struct {
|
||||||
inhub bool
|
inhub bool
|
||||||
fname string
|
fname string
|
||||||
|
@ -127,7 +128,7 @@ func (h *Hub) getItemFileInfo(path string) (*itemFileInfo, error) {
|
||||||
return ret, nil
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// sortedVersions returns the input data, sorted in reverse order (new, old) by semver
|
// sortedVersions returns the input data, sorted in reverse order (new, old) by semver.
|
||||||
func sortedVersions(raw []string) ([]string, error) {
|
func sortedVersions(raw []string) ([]string, error) {
|
||||||
vs := make([]*semver.Version, len(raw))
|
vs := make([]*semver.Version, len(raw))
|
||||||
|
|
||||||
|
@ -150,10 +151,14 @@ func sortedVersions(raw []string) ([]string, error) {
|
||||||
return ret, nil
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func newLocalItem(h *Hub, path string, info *itemFileInfo) *Item {
|
func newLocalItem(h *Hub, path string, info *itemFileInfo) (*Item, error) {
|
||||||
|
type localItemName struct {
|
||||||
|
Name string `yaml:"name"`
|
||||||
|
}
|
||||||
|
|
||||||
_, fileName := filepath.Split(path)
|
_, fileName := filepath.Split(path)
|
||||||
|
|
||||||
return &Item{
|
item := &Item{
|
||||||
hub: h,
|
hub: h,
|
||||||
Name: info.fname,
|
Name: info.fname,
|
||||||
Stage: info.stage,
|
Stage: info.stage,
|
||||||
|
@ -165,6 +170,25 @@ func newLocalItem(h *Hub, path string, info *itemFileInfo) *Item {
|
||||||
UpToDate: true,
|
UpToDate: true,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// try to read the name from the file
|
||||||
|
itemName := localItemName{}
|
||||||
|
|
||||||
|
itemContent, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read %s: %w", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = yaml.Unmarshal(itemContent, &itemName)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to unmarshal %s: %w", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if itemName.Name != "" {
|
||||||
|
item.Name = itemName.Name
|
||||||
|
}
|
||||||
|
|
||||||
|
return item, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
|
func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
|
||||||
|
@ -198,7 +222,11 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
|
||||||
|
|
||||||
if !info.inhub {
|
if !info.inhub {
|
||||||
log.Tracef("%s is a local file, skip", path)
|
log.Tracef("%s is a local file, skip", path)
|
||||||
h.Items[info.ftype][info.fname] = newLocalItem(h, path, info)
|
item, err := newLocalItem(h, path, info)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
h.Items[info.ftype][item.Name] = item
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -269,13 +297,13 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkSubItems checks for the presence, taint and version state of sub-items
|
// checkSubItems checks for the presence, taint and version state of sub-items.
|
||||||
func (h *Hub) checkSubItems(v *Item) error {
|
func (h *Hub) checkSubItems(v *Item) error {
|
||||||
if !v.HasSubItems() {
|
if !v.HasSubItems() {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if v.versionStatus() != VersionUpToDate {
|
if v.versionStatus() != versionUpToDate {
|
||||||
log.Debugf("%s dependencies not checked: not up-to-date", v.Name)
|
log.Debugf("%s dependencies not checked: not up-to-date", v.Name)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -321,7 +349,7 @@ func (h *Hub) checkSubItems(v *Item) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// syncDir scans a directory for items, and updates the Hub state accordingly
|
// syncDir scans a directory for items, and updates the Hub state accordingly.
|
||||||
func (h *Hub) syncDir(dir string) error {
|
func (h *Hub) syncDir(dir string) error {
|
||||||
// For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last
|
// For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last
|
||||||
for _, scan := range ItemTypes {
|
for _, scan := range ItemTypes {
|
||||||
|
@ -347,7 +375,7 @@ func (h *Hub) syncDir(dir string) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// insert a string in a sorted slice, case insensitive, and return the new slice
|
// insert a string in a sorted slice, case insensitive, and return the new slice.
|
||||||
func insertInOrderNoCase(sl []string, value string) []string {
|
func insertInOrderNoCase(sl []string, value string) []string {
|
||||||
i := sort.Search(len(sl), func(i int) bool {
|
i := sort.Search(len(sl), func(i int) bool {
|
||||||
return strings.ToLower(sl[i]) >= strings.ToLower(value)
|
return strings.ToLower(sl[i]) >= strings.ToLower(value)
|
||||||
|
@ -356,7 +384,7 @@ func insertInOrderNoCase(sl []string, value string) []string {
|
||||||
return append(sl[:i], append([]string{value}, sl[i:]...)...)
|
return append(sl[:i], append([]string{value}, sl[i:]...)...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// localSync updates the hub state with downloaded, installed and local items
|
// localSync updates the hub state with downloaded, installed and local items.
|
||||||
func (h *Hub) localSync() error {
|
func (h *Hub) localSync() error {
|
||||||
err := h.syncDir(h.local.InstallDir)
|
err := h.syncDir(h.local.InstallDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -387,17 +415,19 @@ func (h *Hub) localSync() error {
|
||||||
|
|
||||||
vs := item.versionStatus()
|
vs := item.versionStatus()
|
||||||
switch vs {
|
switch vs {
|
||||||
case VersionUpToDate: // latest
|
case versionUpToDate: // latest
|
||||||
if err := h.checkSubItems(item); err != nil {
|
if err := h.checkSubItems(item); err != nil {
|
||||||
warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err))
|
warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err))
|
||||||
}
|
}
|
||||||
case VersionUpdateAvailable: // not up-to-date
|
case versionUpdateAvailable: // not up-to-date
|
||||||
warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version))
|
warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version))
|
||||||
case VersionFuture:
|
case versionFuture:
|
||||||
warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version))
|
warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version))
|
||||||
case VersionUnknown:
|
case versionUnknown:
|
||||||
|
if !item.IsLocal() {
|
||||||
warnings = append(warnings, fmt.Sprintf("collection %s is tainted (latest:%s)", item.Name, item.Version))
|
warnings = append(warnings, fmt.Sprintf("collection %s is tainted (latest:%s)", item.Name, item.Version))
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
log.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.State.LocalVersion, item.Version, item.Versions)
|
log.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.State.LocalVersion, item.Version, item.Versions)
|
||||||
}
|
}
|
||||||
|
|
|
@ -107,3 +107,43 @@ teardown() {
|
||||||
refute_output
|
refute_output
|
||||||
refute_stderr
|
refute_stderr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@test "a local item is not tainted" {
|
||||||
|
# not from cscli... inspect
|
||||||
|
rune -0 mkdir -p "$CONFIG_DIR/collections"
|
||||||
|
rune -0 touch "$CONFIG_DIR/collections/foobar.yaml"
|
||||||
|
rune -0 cscli collections inspect foobar.yaml -o json
|
||||||
|
rune -0 jq -e '.tainted==false' <(output)
|
||||||
|
|
||||||
|
rune -0 cscli collections install crowdsecurity/sshd
|
||||||
|
rune -0 truncate -s0 "$CONFIG_DIR/collections/sshd.yaml"
|
||||||
|
rune -0 cscli collections inspect crowdsecurity/sshd -o json
|
||||||
|
rune -0 jq -e '.tainted==true' <(output)
|
||||||
|
|
||||||
|
# and not from hub update
|
||||||
|
rune -0 cscli hub update
|
||||||
|
assert_stderr --partial "collection crowdsecurity/sshd is tainted"
|
||||||
|
refute_stderr --partial "collection foobar.yaml is tainted"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "a local item's name defaults to its filename" {
|
||||||
|
rune -0 mkdir -p "$CONFIG_DIR/collections"
|
||||||
|
rune -0 touch "$CONFIG_DIR/collections/foobar.yaml"
|
||||||
|
rune -0 cscli collections list -o json
|
||||||
|
rune -0 jq -r '.[][].name' <(output)
|
||||||
|
assert_output "foobar.yaml"
|
||||||
|
rune -0 cscli collections list foobar.yaml
|
||||||
|
rune -0 cscli collections inspect foobar.yaml -o json
|
||||||
|
rune -0 jq -e '.installed==true' <(output)
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "a local item can provide its own name" {
|
||||||
|
rune -0 mkdir -p "$CONFIG_DIR/collections"
|
||||||
|
echo "name: hi-its-me" > "$CONFIG_DIR/collections/foobar.yaml"
|
||||||
|
rune -0 cscli collections list -o json
|
||||||
|
rune -0 jq -r '.[][].name' <(output)
|
||||||
|
assert_output "hi-its-me"
|
||||||
|
rune -0 cscli collections list hi-its-me
|
||||||
|
rune -0 cscli collections inspect hi-its-me -o json
|
||||||
|
rune -0 jq -e '.installed==true' <(output)
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in a new issue