pkg/cwhub documentation (#2607)

* pkg/cwhub: package documentation

* Don't repeat local state in "cscli... inspect"

* lint

* use proper name of the hub item instead of the filename for local items

* hub update: avoid reporting local items as tainted
This commit is contained in:
mmetc 2023-11-21 17:43:10 +01:00 committed by GitHub
parent 1509c2d97c
commit 2c652ef92f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 299 additions and 118 deletions

View file

@ -268,7 +268,7 @@ func NewItemsInstallCmd(typeName string) *cobra.Command {
func istalledParentNames(item *cwhub.Item) []string {
ret := make([]string, 0)
for _, parent := range item.ParentCollections() {
for _, parent := range item.AncestorCollections() {
if parent.State.Installed {
ret = append(ret, parent.Name)
}

View file

@ -1,7 +1,3 @@
// Package cwhub is responsible for installing and upgrading the local hub files.
//
// This includes retrieving the index, the items to install (parsers, scenarios, data files...)
// and managing the dependencies and taints.
package cwhub
import (

View file

@ -27,7 +27,7 @@ const mockURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s"
var responseByPath map[string]string
// testHub initializes a temporary hub with an empty json file, optionally updating it
// testHub initializes a temporary hub with an empty json file, optionally updating it.
func testHub(t *testing.T, update bool) *Hub {
tmpDir, err := os.MkdirTemp("", "testhub")
require.NoError(t, err)
@ -67,7 +67,7 @@ func testHub(t *testing.T, update bool) *Hub {
return hub
}
// envSetup initializes the temporary hub and mocks the http client
// envSetup initializes the temporary hub and mocks the http client.
func envSetup(t *testing.T) *Hub {
setResponseByPath()
log.SetLevel(log.DebugLevel)
@ -92,7 +92,7 @@ func newMockTransport() http.RoundTripper {
return &mockTransport{}
}
// Implement http.RoundTripper
// Implement http.RoundTripper.
func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
// Create mocked http.Response
response := &http.Response{

View file

@ -13,11 +13,12 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/types"
)
// The DataSet is a list of data sources required by an item (built from the data: section in the yaml).
type DataSet struct {
Data []types.DataSource `yaml:"data,omitempty"`
}
// downloadFile downloads a file and writes it to disk, with no hash verification
// downloadFile downloads a file and writes it to disk, with no hash verification.
func downloadFile(url string, destPath string) error {
log.Debugf("downloading %s in %s", url, destPath)
@ -50,7 +51,7 @@ func downloadFile(url string, destPath string) error {
return nil
}
// downloadDataSet downloads all the data files for an item
// downloadDataSet downloads all the data files for an item.
func downloadDataSet(dataFolder string, force bool, reader io.Reader) error {
dec := yaml.NewDecoder(reader)

113
pkg/cwhub/doc.go Normal file
View file

@ -0,0 +1,113 @@
// Package cwhub is responsible for installing and upgrading the local hub files for CrowdSec.
//
// # Definitions
//
// - A hub ITEM is a file that defines a parser, a scenario, a collection... in the case of a collection, it has dependencies on other hub items.
// - The hub INDEX is a JSON file that contains a tree of available hub items.
// - A REMOTE HUB is an HTTP server that hosts the hub index and the hub items. It can serve from several branches, usually linked to the CrowdSec version.
// - A LOCAL HUB is a directory that contains a copy of the hub index and the downloaded hub items.
//
// Once downloaded, hub items can be installed by linking to them from the configuration directory.
// If an item is present in the configuration directory but it's not a link to the local hub, it is
// considered as a LOCAL ITEM and won't be removed or upgraded.
//
// # Directory Structure
//
// A typical directory layout is the following:
//
// For the local hub (HubDir = /etc/crowdsec/hub):
//
// - /etc/crowdsec/hub/.index.json
// - /etc/crowdsec/hub/parsers/{stage}/{author}/{parser-name}.yaml
// - /etc/crowdsec/hub/scenarios/{author}/{scenario-name}.yaml
//
// For the configuration directory (InstallDir = /etc/crowdsec):
//
// - /etc/crowdsec/parsers/{stage}/{parser-name.yaml} -> /etc/crowdsec/hub/parsers/{stage}/{author}/{parser-name}.yaml
// - /etc/crowdsec/scenarios/{scenario-name.yaml} -> /etc/crowdsec/hub/scenarios/{author}/{scenario-name}.yaml
// - /etc/crowdsec/scenarios/local-scenario.yaml
//
// Note that installed items are not grouped by author, this may change in the future if we want to
// support items with the same name from different authors.
//
// Only parsers and postoverflows have the concept of stage.
//
// Additionally, an item can reference a DATA SET that is installed in a different location than
// the item itself. These files are stored in the data directory (InstallDataDir = /var/lib/crowdsec/data).
//
// - /var/lib/crowdsec/data/http_path_traversal.txt
// - /var/lib/crowdsec/data/jira_cve_2021-26086.txt
// - /var/lib/crowdsec/data/log4j2_cve_2021_44228.txt
// - /var/lib/crowdsec/data/sensitive_data.txt
//
//
// # Using the package
//
// The main entry point is the Hub struct. You can create a new instance with NewHub().
// This constructor takes three parameters, but only the LOCAL HUB configuration is required:
//
// import (
// "fmt"
// "github.com/crowdsecurity/crowdsec/pkg/csconfig"
// "github.com/crowdsecurity/crowdsec/pkg/cwhub"
// )
//
// localHub := csconfig.LocalHubCfg{
// HubIndexFile: "/etc/crowdsec/hub/.index.json",
// HubDir: "/etc/crowdsec/hub",
// InstallDir: "/etc/crowdsec",
// InstallDataDir: "/var/lib/crowdsec/data",
// }
// hub, err := cwhub.NewHub(localHub, nil, false)
// if err != nil {
// return fmt.Errorf("unable to initialize hub: %w", err)
// }
//
// Now you can use the hub to access the existing items:
//
// // list all the parsers
// for _, parser := range hub.GetItemMap(cwhub.PARSERS) {
// fmt.Printf("parser: %s\n", parser.Name)
// }
//
// // retrieve a specific collection
// coll := hub.GetItem(cwhub.COLLECTIONS, "crowdsecurity/linux")
// if coll == nil {
// return fmt.Errorf("collection not found")
// }
//
// You can also install items if they have already been downloaded:
//
// // install a parser
// force := false
// downloadOnly := false
// err := parser.Install(force, downloadOnly)
// if err != nil {
// return fmt.Errorf("unable to install parser: %w", err)
// }
//
// As soon as you try to install an item that is not downloaded or is not up-to-date (meaning its computed hash
// does not correspond to the latest version available in the index), a download will be attempted and you'll
// get the error "remote hub configuration is not provided".
//
// To provide the remote hub configuration, use the second parameter of NewHub():
//
// remoteHub := cwhub.RemoteHubCfg{
// URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s",
// Branch: "master",
// IndexPath: ".index.json",
// }
// updateIndex := false
// hub, err := cwhub.NewHub(localHub, remoteHub, updateIndex)
// if err != nil {
// return fmt.Errorf("unable to initialize hub: %w", err)
// }
//
// The URLTemplate is a string that will be used to build the URL of the remote hub. It must contain two
// placeholders: the branch and the file path (it will be an index or an item).
//
// Setting the third parameter to true will download the latest version of the index, if available on the
// specified branch.
// There is no exported method to update the index once the hub struct is created.
//
package cwhub

View file

@ -11,8 +11,8 @@ import (
)
// installPath returns the location of the symlink to the item in the hub, or the path of the item itself if it's local
// (eg. /etc/crowdsec/collections/xyz.yaml)
// raises an error if the path goes outside of the install dir
// (eg. /etc/crowdsec/collections/xyz.yaml).
// Raises an error if the path goes outside of the install dir.
func (i *Item) installPath() (string, error) {
p := i.Type
if i.Stage != "" {
@ -23,8 +23,8 @@ func (i *Item) installPath() (string, error) {
}
// downloadPath returns the location of the actual config file in the hub
// (eg. /etc/crowdsec/hub/collections/author/xyz.yaml)
// raises an error if the path goes outside of the hub dir
// (eg. /etc/crowdsec/hub/collections/author/xyz.yaml).
// Raises an error if the path goes outside of the hub dir.
func (i *Item) downloadPath() (string, error) {
ret, err := safePath(i.hub.local.HubDir, i.RemotePath)
if err != nil {
@ -34,7 +34,7 @@ func (i *Item) downloadPath() (string, error) {
return ret, nil
}
// makeLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir
// makeLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir.
func (i *Item) createInstallLink() error {
dest, err := i.installPath()
if err != nil {
@ -63,7 +63,7 @@ func (i *Item) createInstallLink() error {
return nil
}
// enable enables the item by creating a symlink to the downloaded content, and also enables sub-items
// enable enables the item by creating a symlink to the downloaded content, and also enables sub-items.
func (i *Item) enable() error {
if i.State.Installed {
if i.State.Tainted {
@ -97,7 +97,7 @@ func (i *Item) enable() error {
return nil
}
// purge removes the actual config file that was downloaded
// purge removes the actual config file that was downloaded.
func (i *Item) purge() error {
if !i.State.Downloaded {
log.Infof("removing %s: not downloaded -- no need to remove", i.Name)
@ -124,7 +124,7 @@ func (i *Item) purge() error {
return nil
}
// removeInstallLink removes the symlink to the downloaded content
// removeInstallLink removes the symlink to the downloaded content.
func (i *Item) removeInstallLink() error {
syml, err := i.installPath()
if err != nil {
@ -166,7 +166,7 @@ func (i *Item) removeInstallLink() error {
return nil
}
// disable removes the install link, and optionally the downloaded content
// disable removes the install link, and optionally the downloaded content.
func (i *Item) disable(purge bool, force bool) error {
// XXX: should return the number of disabled/purged items to inform the upper layer whether to reload or not
err := i.removeInstallLink()

View file

@ -7,10 +7,10 @@ import (
var (
// ErrNilRemoteHub is returned when the remote hub configuration is not provided to the NewHub constructor.
// All attempts to download index or items will return this error.
ErrNilRemoteHub = errors.New("remote hub configuration is not provided. Please report this issue to the developers")
)
// IndexNotFoundError is returned when the remote hub index is not found.
type IndexNotFoundError struct {
URL string
Branch string

View file

@ -19,7 +19,7 @@ import (
"slices"
)
// Install installs the item from the hub, downloading it if needed
// Install installs the item from the hub, downloading it if needed.
func (i *Item) Install(force bool, downloadOnly bool) error {
if downloadOnly && i.State.Downloaded && i.State.UpToDate {
log.Infof("%s is already downloaded and up-to-date", i.Name)
@ -52,7 +52,7 @@ func (i *Item) Install(force bool, downloadOnly bool) error {
return nil
}
// allDependencies returns a list of all (direct or indirect) dependencies of the item
// allDependencies returns a list of all (direct or indirect) dependencies of the item.
func (i *Item) allDependencies() ([]*Item, error) {
var collectSubItems func(item *Item, visited map[*Item]bool, result *[]*Item) error
@ -94,7 +94,7 @@ func (i *Item) allDependencies() ([]*Item, error) {
return ret, nil
}
// Remove disables the item, optionally removing the downloaded content
// Remove disables the item, optionally removing the downloaded content.
func (i *Item) Remove(purge bool, force bool) (bool, error) {
if i.IsLocal() {
return false, fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name)
@ -123,7 +123,7 @@ func (i *Item) Remove(purge bool, force bool) (bool, error) {
// if the sub depends on a collection that is not a direct or indirect dependency
// of the current item, it is not removed
for _, subParent := range sub.ParentCollections() {
for _, subParent := range sub.AncestorCollections() {
if !purge && !subParent.State.Installed {
continue
}
@ -156,7 +156,7 @@ func (i *Item) Remove(purge bool, force bool) (bool, error) {
return removed, nil
}
// Upgrade downloads and applies the last version from the hub
// Upgrade downloads and applies the last version of the item from the hub.
func (i *Item) Upgrade(force bool) (bool, error) {
updated := false
@ -203,7 +203,7 @@ func (i *Item) Upgrade(force bool) (bool, error) {
return updated, nil
}
// downloadLatest downloads the latest version of the item to the hub directory
// downloadLatest downloads the latest version of the item to the hub directory.
func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) {
// XXX: should return the path of the downloaded file (taken from download())
log.Debugf("Downloading %s %s", i.Type, i.Name)
@ -253,7 +253,7 @@ func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) {
return ret, nil
}
// fetch downloads the item from the hub, verifies the hash and returns the content
// fetch downloads the item from the hub, verifies the hash and returns the content.
func (i *Item) fetch() ([]byte, error) {
url, err := i.hub.remote.urlTo(i.RemotePath)
if err != nil {
@ -291,7 +291,7 @@ func (i *Item) fetch() ([]byte, error) {
return body, nil
}
// download downloads the item from the hub and writes it to the hub directory
// download downloads the item from the hub and writes it to the hub directory.
func (i *Item) download(overwrite bool) (string, error) {
// if user didn't --force, don't overwrite local, tainted, up-to-date files
if !overwrite {
@ -348,7 +348,7 @@ func (i *Item) download(overwrite bool) (string, error) {
return finalPath, nil
}
// DownloadDataIfNeeded downloads the data files for the item
// DownloadDataIfNeeded downloads the data set for the item.
func (i *Item) DownloadDataIfNeeded(force bool) error {
itemFilePath, err := i.installPath()
if err != nil {

View file

@ -8,8 +8,8 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection
// We expect the new scenario to be installed
// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection.
// We expect the new scenario to be installed.
func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
hub := envSetup(t)
@ -110,7 +110,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
}
// getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test
// getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test.
func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCfg) *Hub {
hub, err := NewHub(local, remote, false)
require.NoError(t, err, "failed to load hub index")

View file

@ -13,19 +13,22 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
// Hub is the main structure for the package.
type Hub struct {
Items HubItems
Items HubItems // Items read from HubDir and InstallDir
local *csconfig.LocalHubCfg
remote *RemoteHubCfg
Warnings []string
Warnings []string // Warnings encountered during sync
}
// GetDataDir returns the data directory, where data sets are installed.
func (h *Hub) GetDataDir() string {
return h.local.InstallDataDir
}
// NewHub returns a new Hub instance with local and (optionally) remote configuration, and syncs the local state
// It also downloads the index if updateIndex is true
// NewHub returns a new Hub instance with local and (optionally) remote configuration, and syncs the local state.
// If updateIndex is true, the local index file is updated from the remote before reading the state of the items.
// All download operations (including updateIndex) return ErrNilRemoteHub if the remote configuration is not set.
func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, updateIndex bool) (*Hub, error) {
if local == nil {
return nil, fmt.Errorf("no hub configuration found")
@ -55,7 +58,7 @@ func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, updateIndex bool)
return hub, nil
}
// parseIndex takes the content of an index file and fills the map of associated parsers/scenarios/collections
// parseIndex takes the content of an index file and fills the map of associated parsers/scenarios/collections.
func (h *Hub) parseIndex() error {
bidx, err := os.ReadFile(h.local.HubIndexFile)
if err != nil {
@ -91,7 +94,7 @@ func (h *Hub) parseIndex() error {
return nil
}
// ItemStats returns total counts of the hub items
// ItemStats returns total counts of the hub items, including local and tainted.
func (h *Hub) ItemStats() []string {
loaded := ""
local := 0
@ -131,7 +134,7 @@ func (h *Hub) ItemStats() []string {
return ret
}
// updateIndex downloads the latest version of the index and writes it to disk if it changed
// updateIndex downloads the latest version of the index and writes it to disk if it changed.
func (h *Hub) updateIndex() error {
body, err := h.remote.fetchIndex()
if err != nil {

View file

@ -12,7 +12,7 @@ import (
)
const (
// managed item types
// managed item types.
COLLECTIONS = "collections"
PARSERS = "parsers"
POSTOVERFLOWS = "postoverflows"
@ -20,59 +20,57 @@ const (
)
const (
VersionUpToDate = iota
VersionUpdateAvailable
VersionUnknown
VersionFuture
versionUpToDate = iota // the latest version from index is installed
versionUpdateAvailable // not installed, or lower than latest
versionUnknown // local file with no version, or invalid version number
versionFuture // local version is higher latest, but is included in the index: should not happen
)
// The order is important, as it is used to range over sub-items in collections
var ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS}
var (
// The order is important, as it is used to range over sub-items in collections.
ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS}
)
type HubItems map[string]map[string]*Item
// ItemVersion is used to detect the version of a given item
// by comparing the hash of each version to the local file.
// If the item does not match any known version, it is considered tainted.
// If the item does not match any known version, it is considered tainted (modified).
type ItemVersion struct {
Digest string `json:"digest,omitempty"` // meow
Deprecated bool `json:"deprecated,omitempty"`
}
// ItemState is used to keep the local state (i.e. at runtime) of an item
// This data is not stored in the index, but is displayed in the output of "cscli ... inspect"
// ItemState is used to keep the local state (i.e. at runtime) of an item.
// This data is not stored in the index, but is displayed with "cscli ... inspect".
type ItemState struct {
LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR}
LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"`
LocalVersion string `json:"local_version,omitempty"`
LocalHash string `json:"local_hash,omitempty"` // the local meow
LocalHash string `json:"local_hash,omitempty"`
Installed bool `json:"installed"`
Downloaded bool `json:"downloaded"`
UpToDate bool `json:"up_to_date"`
Tainted bool `json:"tainted"` // has it been locally modified?
BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any
Tainted bool `json:"tainted"`
BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"`
}
// Item represents an object managed in the hub. It can be a parser, scenario, collection..
// Item is created from an index file and enriched with local info.
type Item struct {
// back pointer to the hub, to retrieve subitems and call install/remove methods
hub *Hub
hub *Hub // back pointer to the hub, to retrieve other items and call install/remove methods
// local (deployed) info
State ItemState
State ItemState `json:"-" yaml:"-"` // local state, not stored in the index
// descriptive info
Type string `json:"type,omitempty" yaml:"type,omitempty"` // can be any of the ItemTypes
Type string `json:"type,omitempty" yaml:"type,omitempty"` // one of the ItemTypes
Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-...
Name string `json:"name,omitempty"` // as seen in .index.json, usually "author/name"
FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml
Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .index.json
Author string `json:"author,omitempty"` // as seen in .index.json
References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .index.json
Name string `json:"name,omitempty"` // usually "author/name"
FileName string `json:"file_name,omitempty"` // eg. apache2-logs.yaml
Description string `json:"description,omitempty" yaml:"description,omitempty"`
Author string `json:"author,omitempty"`
References []string `json:"references,omitempty" yaml:"references,omitempty"`
// remote (hub) info
RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // the path relative to (git | hub API) ie. /parsers/stage/author/file.yaml
Version string `json:"version,omitempty"` // the last version
Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // the list of existing versions
RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // path relative to the base URL eg. /parsers/stage/author/file.yaml
Version string `json:"version,omitempty"` // the last available version
Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // all the known versions
// if it's a collection, it can have sub items
Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"`
@ -81,17 +79,18 @@ type Item struct {
Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"`
}
// HasSubItems returns true if items of this type can have sub-items. Currently only collections.
func (i *Item) HasSubItems() bool {
return i.Type == COLLECTIONS
}
// IsLocal returns true if the item has been create by a user (not downloaded from the hub).
func (i *Item) IsLocal() bool {
return i.State.Installed && !i.State.Downloaded
}
// MarshalJSON is used to add the "local" field to the json output
// (i.e. with cscli ... inspect -o json)
// It must not use a pointer receiver
// MarshalJSON is used to prepare the output for "cscli ... inspect -o json".
// It must not use a pointer receiver.
func (i Item) MarshalJSON() ([]byte, error) {
type Alias Item
@ -121,9 +120,8 @@ func (i Item) MarshalJSON() ([]byte, error) {
})
}
// MarshalYAML is used to add the "local" field to the yaml output
// (i.e. with cscli ... inspect -o raw)
// It must not use a pointer receiver
// MarshalYAML is used to prepare the output for "cscli ... inspect -o raw".
// It must not use a pointer receiver.
func (i Item) MarshalYAML() (interface{}, error) {
type Alias Item
@ -138,7 +136,7 @@ func (i Item) MarshalYAML() (interface{}, error) {
}, nil
}
// SubItems returns a slice of sub-item pointers, excluding the ones that were not found
// SubItems returns a slice of sub-items, excluding the ones that were not found.
func (i *Item) SubItems() []*Item {
sub := make([]*Item, 0)
@ -211,8 +209,8 @@ func (i *Item) logMissingSubItems() {
}
}
// ParentCollections returns the list of items (collections) that have this item as a direct dependency
func (i *Item) ParentCollections() []*Item {
// AncestorCollections returns a slice of items (collections) that have this item as a direct or indirect dependency.
func (i *Item) AncestorCollections() []*Item {
ret := make([]*Item, 0)
for _, parentName := range i.State.BelongsToCollections {
@ -228,7 +226,7 @@ func (i *Item) ParentCollections() []*Item {
}
// Status returns the status of the item as a string and an emoji
// ie. "enabled,update-available" and emoji.Warning
// (eg. "enabled,update-available" and emoji.Warning).
func (i *Item) Status() (string, emoji.Emoji) {
status := "disabled"
ok := false
@ -269,47 +267,47 @@ func (i *Item) Status() (string, emoji.Emoji) {
return status, emo
}
// versionStatus: semver requires 'v' prefix
// versionStatus returns the status of the item version compared to the hub version.
// semver requires the 'v' prefix.
func (i *Item) versionStatus() int {
local, err := semver.NewVersion(i.State.LocalVersion)
if err != nil {
return VersionUnknown
return versionUnknown
}
// hub versions are already validated while syncing, ignore errors
latest, _ := semver.NewVersion(i.Version)
if local.LessThan(latest) {
return VersionUpdateAvailable
return versionUpdateAvailable
}
if local.Equal(latest) {
return VersionUpToDate
return versionUpToDate
}
return VersionFuture
return versionFuture
}
// validPath returns true if the (relative) path is allowed for the item
// dirNname: the directory name (ie. crowdsecurity)
// fileName: the filename (ie. apache2-logs.yaml)
// validPath returns true if the (relative) path is allowed for the item.
// dirNname: the directory name (ie. crowdsecurity).
// fileName: the filename (ie. apache2-logs.yaml).
func (i *Item) validPath(dirName, fileName string) bool {
return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml")
}
// GetItemMap returns the map of items for a given type
// GetItemMap returns the map of items for a given type.
func (h *Hub) GetItemMap(itemType string) map[string]*Item {
return h.Items[itemType]
}
// GetItem returns the item from hub based on its type and full name (author/name)
// GetItem returns an item from hub based on its type and full name (author/name).
func (h *Hub) GetItem(itemType string, itemName string) *Item {
return h.GetItemMap(itemType)[itemName]
}
// GetItemNames returns the list of (full) item names for a given type
// ie. for collections: crowdsecurity/apache2 crowdsecurity/nginx
// The names can be used to retrieve the item with GetItem()
// GetItemNames returns a slice of (full) item names for a given type
// (eg. for collections: crowdsecurity/apache2 crowdsecurity/nginx).
func (h *Hub) GetItemNames(itemType string) []string {
m := h.GetItemMap(itemType)
if m == nil {
@ -324,7 +322,7 @@ func (h *Hub) GetItemNames(itemType string) []string {
return names
}
// GetAllItems returns a slice of all the items, installed or not
// GetAllItems returns a slice of all the items of a given type, installed or not.
func (h *Hub) GetAllItems(itemType string) ([]*Item, error) {
items, ok := h.Items[itemType]
if !ok {
@ -343,7 +341,7 @@ func (h *Hub) GetAllItems(itemType string) ([]*Item, error) {
return ret, nil
}
// GetInstalledItems returns the list of installed items
// GetInstalledItems returns a slice of the installed items of a given type.
func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) {
items, ok := h.Items[itemType]
if !ok {
@ -361,7 +359,7 @@ func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) {
return retItems, nil
}
// GetInstalledItemNames returns the names of the installed items
// GetInstalledItemNames returns the names of the installed items of a given type.
func (h *Hub) GetInstalledItemNames(itemType string) ([]string, error) {
items, err := h.GetInstalledItems(itemType)
if err != nil {
@ -377,7 +375,7 @@ func (h *Hub) GetInstalledItemNames(itemType string) ([]string, error) {
return retStr, nil
}
// SortItemSlice sorts a slice of items by name, case insensitive
// SortItemSlice sorts a slice of items by name, case insensitive.
func SortItemSlice(items []*Item) {
sort.Slice(items, func(i, j int) bool {
return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name)

View file

@ -10,7 +10,7 @@ import (
"strings"
)
// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary
// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary.
func itemKey(itemPath string) (string, error) {
f, err := os.Lstat(itemPath)
if err != nil {
@ -37,7 +37,7 @@ func itemKey(itemPath string) (string, error) {
return fmt.Sprintf("%s/%s", author, fname), nil
}
// GetItemByPath retrieves the item from the hub index based on its path.
// GetItemByPath retrieves an item from the hub index based on its local path.
func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) {
itemKey, err := itemKey(itemPath)
if err != nil {

View file

@ -6,14 +6,14 @@ import (
"net/http"
)
// RemoteHubCfg contains where to find the remote hub, which branch etc.
// RemoteHubCfg is used to retrieve index and items from the remote hub.
type RemoteHubCfg struct {
Branch string
URLTemplate string
IndexPath string
}
// urlTo builds the URL to download a file from the remote hub
// urlTo builds the URL to download a file from the remote hub.
func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) {
if r == nil {
return "", ErrNilRemoteHub
@ -27,7 +27,7 @@ func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) {
return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil
}
// fetchIndex downloads the index from the hub and returns the content
// fetchIndex downloads the index from the hub and returns the content.
func (r *RemoteHubCfg) fetchIndex() ([]byte, error) {
if r == nil {
return nil, ErrNilRemoteHub

View file

@ -12,13 +12,14 @@ import (
"github.com/Masterminds/semver/v3"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v3"
)
func isYAMLFileName(path string) bool {
return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")
}
// linkTarget returns the target of a symlink, or empty string if it's dangling
// linkTarget returns the target of a symlink, or empty string if it's dangling.
func linkTarget(path string) (string, error) {
hubpath, err := os.Readlink(path)
if err != nil {
@ -52,7 +53,7 @@ func getSHA256(filepath string) (string, error) {
return hex.EncodeToString(h.Sum(nil)), nil
}
// information used to create a new Item, from a file path
// information used to create a new Item, from a file path.
type itemFileInfo struct {
inhub bool
fname string
@ -127,7 +128,7 @@ func (h *Hub) getItemFileInfo(path string) (*itemFileInfo, error) {
return ret, nil
}
// sortedVersions returns the input data, sorted in reverse order (new, old) by semver
// sortedVersions returns the input data, sorted in reverse order (new, old) by semver.
func sortedVersions(raw []string) ([]string, error) {
vs := make([]*semver.Version, len(raw))
@ -150,10 +151,14 @@ func sortedVersions(raw []string) ([]string, error) {
return ret, nil
}
func newLocalItem(h *Hub, path string, info *itemFileInfo) *Item {
func newLocalItem(h *Hub, path string, info *itemFileInfo) (*Item, error) {
type localItemName struct {
Name string `yaml:"name"`
}
_, fileName := filepath.Split(path)
return &Item{
item := &Item{
hub: h,
Name: info.fname,
Stage: info.stage,
@ -165,6 +170,25 @@ func newLocalItem(h *Hub, path string, info *itemFileInfo) *Item {
UpToDate: true,
},
}
// try to read the name from the file
itemName := localItemName{}
itemContent, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("failed to read %s: %w", path, err)
}
err = yaml.Unmarshal(itemContent, &itemName)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal %s: %w", path, err)
}
if itemName.Name != "" {
item.Name = itemName.Name
}
return item, nil
}
func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
@ -198,7 +222,11 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
if !info.inhub {
log.Tracef("%s is a local file, skip", path)
h.Items[info.ftype][info.fname] = newLocalItem(h, path, info)
item, err := newLocalItem(h, path, info)
if err != nil {
return err
}
h.Items[info.ftype][item.Name] = item
return nil
}
@ -269,13 +297,13 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
return nil
}
// checkSubItems checks for the presence, taint and version state of sub-items
// checkSubItems checks for the presence, taint and version state of sub-items.
func (h *Hub) checkSubItems(v *Item) error {
if !v.HasSubItems() {
return nil
}
if v.versionStatus() != VersionUpToDate {
if v.versionStatus() != versionUpToDate {
log.Debugf("%s dependencies not checked: not up-to-date", v.Name)
return nil
}
@ -321,7 +349,7 @@ func (h *Hub) checkSubItems(v *Item) error {
return nil
}
// syncDir scans a directory for items, and updates the Hub state accordingly
// syncDir scans a directory for items, and updates the Hub state accordingly.
func (h *Hub) syncDir(dir string) error {
// For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last
for _, scan := range ItemTypes {
@ -347,7 +375,7 @@ func (h *Hub) syncDir(dir string) error {
return nil
}
// insert a string in a sorted slice, case insensitive, and return the new slice
// insert a string in a sorted slice, case insensitive, and return the new slice.
func insertInOrderNoCase(sl []string, value string) []string {
i := sort.Search(len(sl), func(i int) bool {
return strings.ToLower(sl[i]) >= strings.ToLower(value)
@ -356,7 +384,7 @@ func insertInOrderNoCase(sl []string, value string) []string {
return append(sl[:i], append([]string{value}, sl[i:]...)...)
}
// localSync updates the hub state with downloaded, installed and local items
// localSync updates the hub state with downloaded, installed and local items.
func (h *Hub) localSync() error {
err := h.syncDir(h.local.InstallDir)
if err != nil {
@ -387,17 +415,19 @@ func (h *Hub) localSync() error {
vs := item.versionStatus()
switch vs {
case VersionUpToDate: // latest
case versionUpToDate: // latest
if err := h.checkSubItems(item); err != nil {
warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err))
}
case VersionUpdateAvailable: // not up-to-date
case versionUpdateAvailable: // not up-to-date
warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version))
case VersionFuture:
case versionFuture:
warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version))
case VersionUnknown:
case versionUnknown:
if !item.IsLocal() {
warnings = append(warnings, fmt.Sprintf("collection %s is tainted (latest:%s)", item.Name, item.Version))
}
}
log.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.State.LocalVersion, item.Version, item.Versions)
}

View file

@ -107,3 +107,43 @@ teardown() {
refute_output
refute_stderr
}
@test "a local item is not tainted" {
# not from cscli... inspect
rune -0 mkdir -p "$CONFIG_DIR/collections"
rune -0 touch "$CONFIG_DIR/collections/foobar.yaml"
rune -0 cscli collections inspect foobar.yaml -o json
rune -0 jq -e '.tainted==false' <(output)
rune -0 cscli collections install crowdsecurity/sshd
rune -0 truncate -s0 "$CONFIG_DIR/collections/sshd.yaml"
rune -0 cscli collections inspect crowdsecurity/sshd -o json
rune -0 jq -e '.tainted==true' <(output)
# and not from hub update
rune -0 cscli hub update
assert_stderr --partial "collection crowdsecurity/sshd is tainted"
refute_stderr --partial "collection foobar.yaml is tainted"
}
@test "a local item's name defaults to its filename" {
rune -0 mkdir -p "$CONFIG_DIR/collections"
rune -0 touch "$CONFIG_DIR/collections/foobar.yaml"
rune -0 cscli collections list -o json
rune -0 jq -r '.[][].name' <(output)
assert_output "foobar.yaml"
rune -0 cscli collections list foobar.yaml
rune -0 cscli collections inspect foobar.yaml -o json
rune -0 jq -e '.installed==true' <(output)
}
@test "a local item can provide its own name" {
rune -0 mkdir -p "$CONFIG_DIR/collections"
echo "name: hi-its-me" > "$CONFIG_DIR/collections/foobar.yaml"
rune -0 cscli collections list -o json
rune -0 jq -r '.[][].name' <(output)
assert_output "hi-its-me"
rune -0 cscli collections list hi-its-me
rune -0 cscli collections inspect hi-its-me -o json
rune -0 jq -e '.installed==true' <(output)
}