From 2c652ef92fedb2a331e56cba3cf6371339ccb81a Mon Sep 17 00:00:00 2001
From: mmetc <92726601+mmetc@users.noreply.github.com>
Date: Tue, 21 Nov 2023 17:43:10 +0100
Subject: [PATCH] pkg/cwhub documentation (#2607)

* pkg/cwhub: package documentation

* Don't repeat local state in "cscli... inspect"

* lint

* use proper name of the hub item instead of the filename for local items

* hub update: avoid reporting local items as tainted
---
 cmd/crowdsec-cli/itemcommands.go |   2 +-
 pkg/cwhub/cwhub.go               |   4 --
 pkg/cwhub/cwhub_test.go          |   6 +-
 pkg/cwhub/dataset.go             |   5 +-
 pkg/cwhub/doc.go                 | 113 ++++++++++++++++++++++++++++++
 pkg/cwhub/enable.go              |  18 ++---
 pkg/cwhub/errors.go              |   2 +-
 pkg/cwhub/helpers.go             |  18 ++---
 pkg/cwhub/helpers_test.go        |   6 +-
 pkg/cwhub/hub.go                 |  17 +++--
 pkg/cwhub/items.go               | 114 +++++++++++++++----------------
 pkg/cwhub/leakybucket.go         |   4 +-
 pkg/cwhub/remote.go              |   6 +-
 pkg/cwhub/sync.go                |  62 ++++++++++++-----
 test/bats/20_hub_items.bats      |  40 +++++++++++
 15 files changed, 299 insertions(+), 118 deletions(-)
 create mode 100644 pkg/cwhub/doc.go

diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcommands.go
index f36f52a48..82fe113c1 100644
--- a/cmd/crowdsec-cli/itemcommands.go
+++ b/cmd/crowdsec-cli/itemcommands.go
@@ -268,7 +268,7 @@ func NewItemsInstallCmd(typeName string) *cobra.Command {
 func istalledParentNames(item *cwhub.Item) []string {
 	ret := make([]string, 0)
 
-	for _, parent := range item.ParentCollections() {
+	for _, parent := range item.AncestorCollections() {
 		if parent.State.Installed {
 			ret = append(ret, parent.Name)
 		}
diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go
index 85569c64e..ff34bed59 100644
--- a/pkg/cwhub/cwhub.go
+++ b/pkg/cwhub/cwhub.go
@@ -1,7 +1,3 @@
-// Package cwhub is responsible for installing and upgrading the local hub files.
-//
-// This includes retrieving the index, the items to install (parsers, scenarios, data files...)
-// and managing the dependencies and taints.
 package cwhub
 
 import (
diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go
index 3a260cb9a..270f003c3 100644
--- a/pkg/cwhub/cwhub_test.go
+++ b/pkg/cwhub/cwhub_test.go
@@ -27,7 +27,7 @@ const mockURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s"
 
 var responseByPath map[string]string
 
-// testHub initializes a temporary hub with an empty json file, optionally updating it
+// testHub initializes a temporary hub with an empty json file, optionally updating it.
 func testHub(t *testing.T, update bool) *Hub {
 	tmpDir, err := os.MkdirTemp("", "testhub")
 	require.NoError(t, err)
@@ -67,7 +67,7 @@ func testHub(t *testing.T, update bool) *Hub {
 	return hub
 }
 
-// envSetup initializes the temporary hub and mocks the http client
+// envSetup initializes the temporary hub and mocks the http client.
 func envSetup(t *testing.T) *Hub {
 	setResponseByPath()
 	log.SetLevel(log.DebugLevel)
@@ -92,7 +92,7 @@ func newMockTransport() http.RoundTripper {
 	return &mockTransport{}
 }
 
-// Implement http.RoundTripper
+// Implement http.RoundTripper.
 func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 	// Create mocked http.Response
 	response := &http.Response{
diff --git a/pkg/cwhub/dataset.go b/pkg/cwhub/dataset.go
index 79eb91573..e624436c8 100644
--- a/pkg/cwhub/dataset.go
+++ b/pkg/cwhub/dataset.go
@@ -13,11 +13,12 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
+// The DataSet is a list of data sources required by an item (built from the data: section in the yaml).
 type DataSet struct {
 	Data []types.DataSource `yaml:"data,omitempty"`
 }
 
-// downloadFile downloads a file and writes it to disk, with no hash verification
+// downloadFile downloads a file and writes it to disk, with no hash verification.
 func downloadFile(url string, destPath string) error {
 	log.Debugf("downloading %s in %s", url, destPath)
 
@@ -50,7 +51,7 @@ func downloadFile(url string, destPath string) error {
 	return nil
 }
 
-// downloadDataSet downloads all the data files for an item
+// downloadDataSet downloads all the data files for an item.
 func downloadDataSet(dataFolder string, force bool, reader io.Reader) error {
 	dec := yaml.NewDecoder(reader)
 
diff --git a/pkg/cwhub/doc.go b/pkg/cwhub/doc.go
new file mode 100644
index 000000000..857672650
--- /dev/null
+++ b/pkg/cwhub/doc.go
@@ -0,0 +1,113 @@
+// Package cwhub is responsible for installing and upgrading the local hub files for CrowdSec.
+//
+// # Definitions
+//
+//  - A hub ITEM is a file that defines a parser, a scenario, a collection... in the case of a collection, it has dependencies on other hub items.
+//  - The hub INDEX is a JSON file that contains a tree of available hub items.
+//  - A REMOTE HUB is an HTTP server that hosts the hub index and the hub items. It can serve from several branches, usually linked to the CrowdSec version.
+//  - A LOCAL HUB is a directory that contains a copy of the hub index and the downloaded hub items.
+//
+// Once downloaded, hub items can be installed by linking to them from the configuration directory.
+// If an item is present in the configuration directory but it's not a link to the local hub, it is
+// considered as a LOCAL ITEM and won't be removed or upgraded.
+//
+// # Directory Structure
+//
+// A typical directory layout is the following:
+//
+// For the local hub (HubDir = /etc/crowdsec/hub):
+//
+//  - /etc/crowdsec/hub/.index.json
+//  - /etc/crowdsec/hub/parsers/{stage}/{author}/{parser-name}.yaml
+//  - /etc/crowdsec/hub/scenarios/{author}/{scenario-name}.yaml
+//
+// For the configuration directory (InstallDir = /etc/crowdsec):
+//
+//  - /etc/crowdsec/parsers/{stage}/{parser-name.yaml} -> /etc/crowdsec/hub/parsers/{stage}/{author}/{parser-name}.yaml
+//  - /etc/crowdsec/scenarios/{scenario-name.yaml} -> /etc/crowdsec/hub/scenarios/{author}/{scenario-name}.yaml
+//  - /etc/crowdsec/scenarios/local-scenario.yaml
+//
+// Note that installed items are not grouped by author, this may change in the future if we want to
+// support items with the same name from different authors.
+//
+// Only parsers and postoverflows have the concept of stage.
+//
+// Additionally, an item can reference a DATA SET that is installed in a different location than
+// the item itself. These files are stored in the data directory (InstallDataDir = /var/lib/crowdsec/data).
+//
+//  - /var/lib/crowdsec/data/http_path_traversal.txt
+//  - /var/lib/crowdsec/data/jira_cve_2021-26086.txt
+//  - /var/lib/crowdsec/data/log4j2_cve_2021_44228.txt
+//  - /var/lib/crowdsec/data/sensitive_data.txt
+//
+//
+// # Using the package
+//
+// The main entry point is the Hub struct. You can create a new instance with NewHub().
+// This constructor takes three parameters, but only the LOCAL HUB configuration is required:
+//
+//	import (
+//		"fmt"
+//		"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+//		"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+//	)
+//
+//	localHub := csconfig.LocalHubCfg{
+//		HubIndexFile:	"/etc/crowdsec/hub/.index.json",
+//		HubDir:		"/etc/crowdsec/hub",
+//		InstallDir:	"/etc/crowdsec",
+//		InstallDataDir: "/var/lib/crowdsec/data",
+//	}
+//	hub, err := cwhub.NewHub(localHub, nil, false)
+//	if err != nil {
+//		return fmt.Errorf("unable to initialize hub: %w", err)
+//	}
+//
+// Now you can use the hub to access the existing items:
+//
+//	// list all the parsers
+//	for _, parser := range hub.GetItemMap(cwhub.PARSERS) {
+//		fmt.Printf("parser: %s\n", parser.Name)
+//	}
+//
+//	// retrieve a specific collection
+//	coll := hub.GetItem(cwhub.COLLECTIONS, "crowdsecurity/linux")
+//	if coll == nil {
+//		return fmt.Errorf("collection not found")
+//	}
+//
+// You can also install items if they have already been downloaded:
+//
+// 	// install a parser
+// 	force := false
+// 	downloadOnly := false
+// 	err := parser.Install(force, downloadOnly)
+// 	if err != nil {
+// 		return fmt.Errorf("unable to install parser: %w", err)
+// 	}
+//
+// As soon as you try to install an item that is not downloaded or is not up-to-date (meaning its computed hash
+// does not correspond to the latest version available in the index), a download will be attempted and you'll
+// get the error "remote hub configuration is not provided".
+//
+// To provide the remote hub configuration, use the second parameter of NewHub():
+//
+// 	remoteHub := cwhub.RemoteHubCfg{
+//		URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s",
+//		Branch: "master",
+//		IndexPath: ".index.json",
+//	}
+//	updateIndex := false
+//	hub, err := cwhub.NewHub(localHub, remoteHub, updateIndex)
+//	if err != nil {
+//		return fmt.Errorf("unable to initialize hub: %w", err)
+//	}
+//
+// The URLTemplate is a string that will be used to build the URL of the remote hub. It must contain two
+// placeholders: the branch and the file path (it will be an index or an item).
+//
+// Setting the third parameter to true will download the latest version of the index, if available on the
+// specified branch.
+// There is no exported method to update the index once the hub struct is created.
+//
+package cwhub
diff --git a/pkg/cwhub/enable.go b/pkg/cwhub/enable.go
index 2a17fc724..d7f8b8b71 100644
--- a/pkg/cwhub/enable.go
+++ b/pkg/cwhub/enable.go
@@ -11,8 +11,8 @@ import (
 )
 
 // installPath returns the location of the symlink to the item in the hub, or the path of the item itself if it's local
-// (eg. /etc/crowdsec/collections/xyz.yaml)
-// raises an error if the path goes outside of the install dir
+// (eg. /etc/crowdsec/collections/xyz.yaml).
+// Raises an error if the path goes outside of the install dir.
 func (i *Item) installPath() (string, error) {
 	p := i.Type
 	if i.Stage != "" {
@@ -23,8 +23,8 @@ func (i *Item) installPath() (string, error) {
 }
 
 // downloadPath returns the location of the actual config file in the hub
-// (eg. /etc/crowdsec/hub/collections/author/xyz.yaml)
-// raises an error if the path goes outside of the hub dir
+// (eg. /etc/crowdsec/hub/collections/author/xyz.yaml).
+// Raises an error if the path goes outside of the hub dir.
 func (i *Item) downloadPath() (string, error) {
 	ret, err := safePath(i.hub.local.HubDir, i.RemotePath)
 	if err != nil {
@@ -34,7 +34,7 @@ func (i *Item) downloadPath() (string, error) {
 	return ret, nil
 }
 
-// makeLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir
+// makeLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir.
 func (i *Item) createInstallLink() error {
 	dest, err := i.installPath()
 	if err != nil {
@@ -63,7 +63,7 @@ func (i *Item) createInstallLink() error {
 	return nil
 }
 
-// enable enables the item by creating a symlink to the downloaded content, and also enables sub-items
+// enable enables the item by creating a symlink to the downloaded content, and also enables sub-items.
 func (i *Item) enable() error {
 	if i.State.Installed {
 		if i.State.Tainted {
@@ -97,7 +97,7 @@ func (i *Item) enable() error {
 	return nil
 }
 
-// purge removes the actual config file that was downloaded
+// purge removes the actual config file that was downloaded.
 func (i *Item) purge() error {
 	if !i.State.Downloaded {
 		log.Infof("removing %s: not downloaded -- no need to remove", i.Name)
@@ -124,7 +124,7 @@ func (i *Item) purge() error {
 	return nil
 }
 
-// removeInstallLink removes the symlink to the downloaded content
+// removeInstallLink removes the symlink to the downloaded content.
 func (i *Item) removeInstallLink() error {
 	syml, err := i.installPath()
 	if err != nil {
@@ -166,7 +166,7 @@ func (i *Item) removeInstallLink() error {
 	return nil
 }
 
-// disable removes the install link, and optionally the downloaded content
+// disable removes the install link, and optionally the downloaded content.
 func (i *Item) disable(purge bool, force bool) error {
 	// XXX: should return the number of disabled/purged items to inform the upper layer whether to reload or not
 	err := i.removeInstallLink()
diff --git a/pkg/cwhub/errors.go b/pkg/cwhub/errors.go
index ec389921c..789c2eced 100644
--- a/pkg/cwhub/errors.go
+++ b/pkg/cwhub/errors.go
@@ -7,10 +7,10 @@ import (
 
 var (
 	// ErrNilRemoteHub is returned when the remote hub configuration is not provided to the NewHub constructor.
-	// All attempts to download index or items will return this error.
 	ErrNilRemoteHub = errors.New("remote hub configuration is not provided. Please report this issue to the developers")
 )
 
+// IndexNotFoundError is returned when the remote hub index is not found.
 type IndexNotFoundError struct {
 	URL    string
 	Branch string
diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go
index 4a03a12d9..335864050 100644
--- a/pkg/cwhub/helpers.go
+++ b/pkg/cwhub/helpers.go
@@ -19,7 +19,7 @@ import (
 	"slices"
 )
 
-// Install installs the item from the hub, downloading it if needed
+// Install installs the item from the hub, downloading it if needed.
 func (i *Item) Install(force bool, downloadOnly bool) error {
 	if downloadOnly && i.State.Downloaded && i.State.UpToDate {
 		log.Infof("%s is already downloaded and up-to-date", i.Name)
@@ -52,7 +52,7 @@ func (i *Item) Install(force bool, downloadOnly bool) error {
 	return nil
 }
 
-// allDependencies returns a list of all (direct or indirect) dependencies of the item
+// allDependencies returns a list of all (direct or indirect) dependencies of the item.
 func (i *Item) allDependencies() ([]*Item, error) {
 	var collectSubItems func(item *Item, visited map[*Item]bool, result *[]*Item) error
 
@@ -94,7 +94,7 @@ func (i *Item) allDependencies() ([]*Item, error) {
 	return ret, nil
 }
 
-// Remove disables the item, optionally removing the downloaded content
+// Remove disables the item, optionally removing the downloaded content.
 func (i *Item) Remove(purge bool, force bool) (bool, error) {
 	if i.IsLocal() {
 		return false, fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name)
@@ -123,7 +123,7 @@ func (i *Item) Remove(purge bool, force bool) (bool, error) {
 
 		// if the sub depends on a collection that is not a direct or indirect dependency
 		// of the current item, it is not removed
-		for _, subParent := range sub.ParentCollections() {
+		for _, subParent := range sub.AncestorCollections() {
 			if !purge && !subParent.State.Installed {
 				continue
 			}
@@ -156,7 +156,7 @@ func (i *Item) Remove(purge bool, force bool) (bool, error) {
 	return removed, nil
 }
 
-// Upgrade downloads and applies the last version from the hub
+// Upgrade downloads and applies the last version of the item from the hub.
 func (i *Item) Upgrade(force bool) (bool, error) {
 	updated := false
 
@@ -203,7 +203,7 @@ func (i *Item) Upgrade(force bool) (bool, error) {
 	return updated, nil
 }
 
-// downloadLatest downloads the latest version of the item to the hub directory
+// downloadLatest downloads the latest version of the item to the hub directory.
 func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) {
 	// XXX: should return the path of the downloaded file (taken from download())
 	log.Debugf("Downloading %s %s", i.Type, i.Name)
@@ -253,7 +253,7 @@ func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) {
 	return ret, nil
 }
 
-// fetch downloads the item from the hub, verifies the hash and returns the content
+// fetch downloads the item from the hub, verifies the hash and returns the content.
 func (i *Item) fetch() ([]byte, error) {
 	url, err := i.hub.remote.urlTo(i.RemotePath)
 	if err != nil {
@@ -291,7 +291,7 @@ func (i *Item) fetch() ([]byte, error) {
 	return body, nil
 }
 
-// download downloads the item from the hub and writes it to the hub directory
+// download downloads the item from the hub and writes it to the hub directory.
 func (i *Item) download(overwrite bool) (string, error) {
 	// if user didn't --force, don't overwrite local, tainted, up-to-date files
 	if !overwrite {
@@ -348,7 +348,7 @@ func (i *Item) download(overwrite bool) (string, error) {
 	return finalPath, nil
 }
 
-// DownloadDataIfNeeded downloads the data files for the item
+// DownloadDataIfNeeded downloads the data set for the item.
 func (i *Item) DownloadDataIfNeeded(force bool) error {
 	itemFilePath, err := i.installPath()
 	if err != nil {
diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go
index 58d1a56cc..9f8f8c1d7 100644
--- a/pkg/cwhub/helpers_test.go
+++ b/pkg/cwhub/helpers_test.go
@@ -8,8 +8,8 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 )
 
-// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection
-// We expect the new scenario to be installed
+// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection.
+// We expect the new scenario to be installed.
 func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
 	hub := envSetup(t)
 
@@ -110,7 +110,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
 	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
 }
 
-// getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test
+// getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test.
 func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCfg) *Hub {
 	hub, err := NewHub(local, remote, false)
 	require.NoError(t, err, "failed to load hub index")
diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go
index 01f7eef04..ff1c3cf15 100644
--- a/pkg/cwhub/hub.go
+++ b/pkg/cwhub/hub.go
@@ -13,19 +13,22 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 )
 
+// Hub is the main structure for the package.
 type Hub struct {
-	Items    HubItems
+	Items    HubItems // Items read from HubDir and InstallDir
 	local    *csconfig.LocalHubCfg
 	remote   *RemoteHubCfg
-	Warnings []string
+	Warnings []string // Warnings encountered during sync
 }
 
+// GetDataDir returns the data directory, where data sets are installed.
 func (h *Hub) GetDataDir() string {
 	return h.local.InstallDataDir
 }
 
-// NewHub returns a new Hub instance with local and (optionally) remote configuration, and syncs the local state
-// It also downloads the index if updateIndex is true
+// NewHub returns a new Hub instance with local and (optionally) remote configuration, and syncs the local state.
+// If updateIndex is true, the local index file is updated from the remote before reading the state of the items.
+// All download operations (including updateIndex) return ErrNilRemoteHub if the remote configuration is not set.
 func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, updateIndex bool) (*Hub, error) {
 	if local == nil {
 		return nil, fmt.Errorf("no hub configuration found")
@@ -55,7 +58,7 @@ func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, updateIndex bool)
 	return hub, nil
 }
 
-// parseIndex takes the content of an index file and fills the map of associated parsers/scenarios/collections
+// parseIndex takes the content of an index file and fills the map of associated parsers/scenarios/collections.
 func (h *Hub) parseIndex() error {
 	bidx, err := os.ReadFile(h.local.HubIndexFile)
 	if err != nil {
@@ -91,7 +94,7 @@ func (h *Hub) parseIndex() error {
 	return nil
 }
 
-// ItemStats returns total counts of the hub items
+// ItemStats returns total counts of the hub items, including local and tainted.
 func (h *Hub) ItemStats() []string {
 	loaded := ""
 	local := 0
@@ -131,7 +134,7 @@ func (h *Hub) ItemStats() []string {
 	return ret
 }
 
-// updateIndex downloads the latest version of the index and writes it to disk if it changed
+// updateIndex downloads the latest version of the index and writes it to disk if it changed.
 func (h *Hub) updateIndex() error {
 	body, err := h.remote.fetchIndex()
 	if err != nil {
diff --git a/pkg/cwhub/items.go b/pkg/cwhub/items.go
index 7e0a10f4c..1c82adb6f 100644
--- a/pkg/cwhub/items.go
+++ b/pkg/cwhub/items.go
@@ -12,7 +12,7 @@ import (
 )
 
 const (
-	// managed item types
+	// managed item types.
 	COLLECTIONS   = "collections"
 	PARSERS       = "parsers"
 	POSTOVERFLOWS = "postoverflows"
@@ -20,59 +20,57 @@ const (
 )
 
 const (
-	VersionUpToDate = iota
-	VersionUpdateAvailable
-	VersionUnknown
-	VersionFuture
+	versionUpToDate        = iota // the latest version from index is installed
+	versionUpdateAvailable        // not installed, or lower than latest
+	versionUnknown                // local file with no version, or invalid version number
+	versionFuture                 // local version is higher latest, but is included in the index: should not happen
 )
 
-// The order is important, as it is used to range over sub-items in collections
-var ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS}
+var (
+	// The order is important, as it is used to range over sub-items in collections.
+	ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS}
+)
 
 type HubItems map[string]map[string]*Item
 
 // ItemVersion is used to detect the version of a given item
 // by comparing the hash of each version to the local file.
-// If the item does not match any known version, it is considered tainted.
+// If the item does not match any known version, it is considered tainted (modified).
 type ItemVersion struct {
 	Digest     string `json:"digest,omitempty"` // meow
 	Deprecated bool   `json:"deprecated,omitempty"`
 }
 
-// ItemState is used to keep the local state (i.e. at runtime) of an item
-// This data is not stored in the index, but is displayed in the output of "cscli ... inspect"
+// ItemState is used to keep the local state (i.e. at runtime) of an item.
+// This data is not stored in the index, but is displayed with "cscli ... inspect".
 type ItemState struct {
-	LocalPath            string   `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR}
+	LocalPath            string   `json:"local_path,omitempty" yaml:"local_path,omitempty"`
 	LocalVersion         string   `json:"local_version,omitempty"`
-	LocalHash            string   `json:"local_hash,omitempty"` // the local meow
+	LocalHash            string   `json:"local_hash,omitempty"`
 	Installed            bool     `json:"installed"`
 	Downloaded           bool     `json:"downloaded"`
 	UpToDate             bool     `json:"up_to_date"`
-	Tainted              bool     `json:"tainted"`                                                                  // has it been locally modified?
-	BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any
+	Tainted              bool     `json:"tainted"`
+	BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"`
 }
 
-// Item represents an object managed in the hub. It can be a parser, scenario, collection..
+// Item is created from an index file and enriched with local info.
 type Item struct {
-	// back pointer to the hub, to retrieve subitems and call install/remove methods
-	hub *Hub
+	hub *Hub // back pointer to the hub, to retrieve other items and call install/remove methods
 
-	// local (deployed) info
-	State ItemState
+	State ItemState `json:"-" yaml:"-"` // local state, not stored in the index
 
-	// descriptive info
-	Type        string   `json:"type,omitempty"                   yaml:"type,omitempty"`        // can be any of the ItemTypes
-	Stage       string   `json:"stage,omitempty"                  yaml:"stage,omitempty"`       // Stage for parser|postoverflow: s00-raw/s01-...
-	Name        string   `json:"name,omitempty"`                                                // as seen in .index.json, usually "author/name"
-	FileName    string   `json:"file_name,omitempty"`                                           // the filename, ie. apache2-logs.yaml
-	Description string   `json:"description,omitempty"            yaml:"description,omitempty"` // as seen in .index.json
-	Author      string   `json:"author,omitempty"`                                              // as seen in .index.json
-	References  []string `json:"references,omitempty"             yaml:"references,omitempty"`  // as seen in .index.json
+	Type        string   `json:"type,omitempty"                   yaml:"type,omitempty"`  // one of the ItemTypes
+	Stage       string   `json:"stage,omitempty"                  yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-...
+	Name        string   `json:"name,omitempty"`                                          // usually "author/name"
+	FileName    string   `json:"file_name,omitempty"`                                     // eg. apache2-logs.yaml
+	Description string   `json:"description,omitempty"            yaml:"description,omitempty"`
+	Author      string   `json:"author,omitempty"`
+	References  []string `json:"references,omitempty"             yaml:"references,omitempty"`
 
-	// remote (hub) info
-	RemotePath string                 `json:"path,omitempty"      yaml:"remote_path,omitempty"` // the path relative to (git | hub API) ie. /parsers/stage/author/file.yaml
-	Version    string                 `json:"version,omitempty"`                                // the last version
-	Versions   map[string]ItemVersion `json:"versions,omitempty"  yaml:"-"`                     // the list of existing versions
+	RemotePath string                 `json:"path,omitempty"      yaml:"remote_path,omitempty"` // path relative to the base URL eg. /parsers/stage/author/file.yaml
+	Version    string                 `json:"version,omitempty"`                                // the last available version
+	Versions   map[string]ItemVersion `json:"versions,omitempty"  yaml:"-"`                     // all the known versions
 
 	// if it's a collection, it can have sub items
 	Parsers       []string `json:"parsers,omitempty"       yaml:"parsers,omitempty"`
@@ -81,17 +79,18 @@ type Item struct {
 	Collections   []string `json:"collections,omitempty"   yaml:"collections,omitempty"`
 }
 
+// HasSubItems returns true if items of this type can have sub-items. Currently only collections.
 func (i *Item) HasSubItems() bool {
 	return i.Type == COLLECTIONS
 }
 
+// IsLocal returns true if the item has been create by a user (not downloaded from the hub).
 func (i *Item) IsLocal() bool {
 	return i.State.Installed && !i.State.Downloaded
 }
 
-// MarshalJSON is used to add the "local" field to the json output
-// (i.e. with cscli ... inspect -o json)
-// It must not use a pointer receiver
+// MarshalJSON is used to prepare the output for "cscli ... inspect -o json".
+// It must not use a pointer receiver.
 func (i Item) MarshalJSON() ([]byte, error) {
 	type Alias Item
 
@@ -121,9 +120,8 @@ func (i Item) MarshalJSON() ([]byte, error) {
 	})
 }
 
-// MarshalYAML is used to add the "local" field to the yaml output
-// (i.e. with cscli ... inspect -o raw)
-// It must not use a pointer receiver
+// MarshalYAML is used to prepare the output for "cscli ... inspect -o raw".
+// It must not use a pointer receiver.
 func (i Item) MarshalYAML() (interface{}, error) {
 	type Alias Item
 
@@ -138,7 +136,7 @@ func (i Item) MarshalYAML() (interface{}, error) {
 	}, nil
 }
 
-// SubItems returns a slice of sub-item pointers, excluding the ones that were not found
+// SubItems returns a slice of sub-items, excluding the ones that were not found.
 func (i *Item) SubItems() []*Item {
 	sub := make([]*Item, 0)
 
@@ -211,8 +209,8 @@ func (i *Item) logMissingSubItems() {
 	}
 }
 
-// ParentCollections returns the list of items (collections) that have this item as a direct dependency
-func (i *Item) ParentCollections() []*Item {
+// AncestorCollections returns a slice of items (collections) that have this item as a direct or indirect dependency.
+func (i *Item) AncestorCollections() []*Item {
 	ret := make([]*Item, 0)
 
 	for _, parentName := range i.State.BelongsToCollections {
@@ -228,7 +226,7 @@ func (i *Item) ParentCollections() []*Item {
 }
 
 // Status returns the status of the item as a string and an emoji
-// ie. "enabled,update-available" and emoji.Warning
+// (eg. "enabled,update-available" and emoji.Warning).
 func (i *Item) Status() (string, emoji.Emoji) {
 	status := "disabled"
 	ok := false
@@ -269,47 +267,47 @@ func (i *Item) Status() (string, emoji.Emoji) {
 	return status, emo
 }
 
-// versionStatus: semver requires 'v' prefix
+// versionStatus returns the status of the item version compared to the hub version.
+// semver requires the 'v' prefix.
 func (i *Item) versionStatus() int {
 	local, err := semver.NewVersion(i.State.LocalVersion)
 	if err != nil {
-		return VersionUnknown
+		return versionUnknown
 	}
 
 	// hub versions are already validated while syncing, ignore errors
 	latest, _ := semver.NewVersion(i.Version)
 
 	if local.LessThan(latest) {
-		return VersionUpdateAvailable
+		return versionUpdateAvailable
 	}
 
 	if local.Equal(latest) {
-		return VersionUpToDate
+		return versionUpToDate
 	}
 
-	return VersionFuture
+	return versionFuture
 }
 
-// validPath returns true if the (relative) path is allowed for the item
-// dirNname: the directory name (ie. crowdsecurity)
-// fileName: the filename (ie. apache2-logs.yaml)
+// validPath returns true if the (relative) path is allowed for the item.
+// dirNname: the directory name (ie. crowdsecurity).
+// fileName: the filename (ie. apache2-logs.yaml).
 func (i *Item) validPath(dirName, fileName string) bool {
 	return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml")
 }
 
-// GetItemMap returns the map of items for a given type
+// GetItemMap returns the map of items for a given type.
 func (h *Hub) GetItemMap(itemType string) map[string]*Item {
 	return h.Items[itemType]
 }
 
-// GetItem returns the item from hub based on its type and full name (author/name)
+// GetItem returns an item from hub based on its type and full name (author/name).
 func (h *Hub) GetItem(itemType string, itemName string) *Item {
 	return h.GetItemMap(itemType)[itemName]
 }
 
-// GetItemNames returns the list of (full) item names for a given type
-// ie. for collections: crowdsecurity/apache2 crowdsecurity/nginx
-// The names can be used to retrieve the item with GetItem()
+// GetItemNames returns a slice of (full) item names for a given type
+// (eg. for collections: crowdsecurity/apache2 crowdsecurity/nginx).
 func (h *Hub) GetItemNames(itemType string) []string {
 	m := h.GetItemMap(itemType)
 	if m == nil {
@@ -324,7 +322,7 @@ func (h *Hub) GetItemNames(itemType string) []string {
 	return names
 }
 
-// GetAllItems returns a slice of all the items, installed or not
+// GetAllItems returns a slice of all the items of a given type, installed or not.
 func (h *Hub) GetAllItems(itemType string) ([]*Item, error) {
 	items, ok := h.Items[itemType]
 	if !ok {
@@ -343,7 +341,7 @@ func (h *Hub) GetAllItems(itemType string) ([]*Item, error) {
 	return ret, nil
 }
 
-// GetInstalledItems returns the list of installed items
+// GetInstalledItems returns a slice of the installed items of a given type.
 func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) {
 	items, ok := h.Items[itemType]
 	if !ok {
@@ -361,7 +359,7 @@ func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) {
 	return retItems, nil
 }
 
-// GetInstalledItemNames returns the names of the installed items
+// GetInstalledItemNames returns the names of the installed items of a given type.
 func (h *Hub) GetInstalledItemNames(itemType string) ([]string, error) {
 	items, err := h.GetInstalledItems(itemType)
 	if err != nil {
@@ -377,7 +375,7 @@ func (h *Hub) GetInstalledItemNames(itemType string) ([]string, error) {
 	return retStr, nil
 }
 
-// SortItemSlice sorts a slice of items by name, case insensitive
+// SortItemSlice sorts a slice of items by name, case insensitive.
 func SortItemSlice(items []*Item) {
 	sort.Slice(items, func(i, j int) bool {
 		return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name)
diff --git a/pkg/cwhub/leakybucket.go b/pkg/cwhub/leakybucket.go
index 5d97153cb..8143e9433 100644
--- a/pkg/cwhub/leakybucket.go
+++ b/pkg/cwhub/leakybucket.go
@@ -10,7 +10,7 @@ import (
 	"strings"
 )
 
-// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary
+// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary.
 func itemKey(itemPath string) (string, error) {
 	f, err := os.Lstat(itemPath)
 	if err != nil {
@@ -37,7 +37,7 @@ func itemKey(itemPath string) (string, error) {
 	return fmt.Sprintf("%s/%s", author, fname), nil
 }
 
-// GetItemByPath retrieves the item from the hub index based on its path.
+// GetItemByPath retrieves an item from the hub index based on its local path.
 func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) {
 	itemKey, err := itemKey(itemPath)
 	if err != nil {
diff --git a/pkg/cwhub/remote.go b/pkg/cwhub/remote.go
index c98dfa8f5..c1eb5a708 100644
--- a/pkg/cwhub/remote.go
+++ b/pkg/cwhub/remote.go
@@ -6,14 +6,14 @@ import (
 	"net/http"
 )
 
-// RemoteHubCfg contains where to find the remote hub, which branch etc.
+// RemoteHubCfg is used to retrieve index and items from the remote hub.
 type RemoteHubCfg struct {
 	Branch      string
 	URLTemplate string
 	IndexPath   string
 }
 
-// urlTo builds the URL to download a file from the remote hub
+// urlTo builds the URL to download a file from the remote hub.
 func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) {
 	if r == nil {
 		return "", ErrNilRemoteHub
@@ -27,7 +27,7 @@ func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) {
 	return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil
 }
 
-// fetchIndex downloads the index from the hub and returns the content
+// fetchIndex downloads the index from the hub and returns the content.
 func (r *RemoteHubCfg) fetchIndex() ([]byte, error) {
 	if r == nil {
 		return nil, ErrNilRemoteHub
diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go
index 4325017b4..d5d741015 100644
--- a/pkg/cwhub/sync.go
+++ b/pkg/cwhub/sync.go
@@ -12,13 +12,14 @@ import (
 
 	"github.com/Masterminds/semver/v3"
 	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v3"
 )
 
 func isYAMLFileName(path string) bool {
 	return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")
 }
 
-// linkTarget returns the target of a symlink, or empty string if it's dangling
+// linkTarget returns the target of a symlink, or empty string if it's dangling.
 func linkTarget(path string) (string, error) {
 	hubpath, err := os.Readlink(path)
 	if err != nil {
@@ -52,7 +53,7 @@ func getSHA256(filepath string) (string, error) {
 	return hex.EncodeToString(h.Sum(nil)), nil
 }
 
-// information used to create a new Item, from a file path
+// information used to create a new Item, from a file path.
 type itemFileInfo struct {
 	inhub   bool
 	fname   string
@@ -127,7 +128,7 @@ func (h *Hub) getItemFileInfo(path string) (*itemFileInfo, error) {
 	return ret, nil
 }
 
-// sortedVersions returns the input data, sorted in reverse order (new, old) by semver
+// sortedVersions returns the input data, sorted in reverse order (new, old) by semver.
 func sortedVersions(raw []string) ([]string, error) {
 	vs := make([]*semver.Version, len(raw))
 
@@ -150,10 +151,14 @@ func sortedVersions(raw []string) ([]string, error) {
 	return ret, nil
 }
 
-func newLocalItem(h *Hub, path string, info *itemFileInfo) *Item {
+func newLocalItem(h *Hub, path string, info *itemFileInfo) (*Item, error) {
+	type localItemName struct {
+		Name string `yaml:"name"`
+	}
+
 	_, fileName := filepath.Split(path)
 
-	return &Item{
+	item := &Item{
 		hub:      h,
 		Name:     info.fname,
 		Stage:    info.stage,
@@ -165,6 +170,25 @@ func newLocalItem(h *Hub, path string, info *itemFileInfo) *Item {
 			UpToDate:  true,
 		},
 	}
+
+	// try to read the name from the file
+	itemName := localItemName{}
+
+	itemContent, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read %s: %w", path, err)
+	}
+
+	err = yaml.Unmarshal(itemContent, &itemName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal %s: %w", path, err)
+	}
+
+	if itemName.Name != "" {
+		item.Name = itemName.Name
+	}
+
+	return item, nil
 }
 
 func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
@@ -198,7 +222,11 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
 
 		if !info.inhub {
 			log.Tracef("%s is a local file, skip", path)
-			h.Items[info.ftype][info.fname] = newLocalItem(h, path, info)
+			item, err := newLocalItem(h, path, info)
+			if err != nil {
+				return err
+			}
+			h.Items[info.ftype][item.Name] = item
 
 			return nil
 		}
@@ -269,13 +297,13 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
 	return nil
 }
 
-// checkSubItems checks for the presence, taint and version state of sub-items
+// checkSubItems checks for the presence, taint and version state of sub-items.
 func (h *Hub) checkSubItems(v *Item) error {
 	if !v.HasSubItems() {
 		return nil
 	}
 
-	if v.versionStatus() != VersionUpToDate {
+	if v.versionStatus() != versionUpToDate {
 		log.Debugf("%s dependencies not checked: not up-to-date", v.Name)
 		return nil
 	}
@@ -321,7 +349,7 @@ func (h *Hub) checkSubItems(v *Item) error {
 	return nil
 }
 
-// syncDir scans a directory for items, and updates the Hub state accordingly
+// syncDir scans a directory for items, and updates the Hub state accordingly.
 func (h *Hub) syncDir(dir string) error {
 	// For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last
 	for _, scan := range ItemTypes {
@@ -347,7 +375,7 @@ func (h *Hub) syncDir(dir string) error {
 	return nil
 }
 
-// insert a string in a sorted slice, case insensitive, and return the new slice
+// insert a string in a sorted slice, case insensitive, and return the new slice.
 func insertInOrderNoCase(sl []string, value string) []string {
 	i := sort.Search(len(sl), func(i int) bool {
 		return strings.ToLower(sl[i]) >= strings.ToLower(value)
@@ -356,7 +384,7 @@ func insertInOrderNoCase(sl []string, value string) []string {
 	return append(sl[:i], append([]string{value}, sl[i:]...)...)
 }
 
-// localSync updates the hub state with downloaded, installed and local items
+// localSync updates the hub state with downloaded, installed and local items.
 func (h *Hub) localSync() error {
 	err := h.syncDir(h.local.InstallDir)
 	if err != nil {
@@ -387,16 +415,18 @@ func (h *Hub) localSync() error {
 
 		vs := item.versionStatus()
 		switch vs {
-		case VersionUpToDate: // latest
+		case versionUpToDate: // latest
 			if err := h.checkSubItems(item); err != nil {
 				warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err))
 			}
-		case VersionUpdateAvailable: // not up-to-date
+		case versionUpdateAvailable: // not up-to-date
 			warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version))
-		case VersionFuture:
+		case versionFuture:
 			warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version))
-		case VersionUnknown:
-			warnings = append(warnings, fmt.Sprintf("collection %s is tainted (latest:%s)", item.Name, item.Version))
+		case versionUnknown:
+			if !item.IsLocal() {
+				warnings = append(warnings, fmt.Sprintf("collection %s is tainted (latest:%s)", item.Name, item.Version))
+			}
 		}
 
 		log.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.State.LocalVersion, item.Version, item.Versions)
diff --git a/test/bats/20_hub_items.bats b/test/bats/20_hub_items.bats
index 1a795e625..73f2bbcb9 100644
--- a/test/bats/20_hub_items.bats
+++ b/test/bats/20_hub_items.bats
@@ -107,3 +107,43 @@ teardown() {
     refute_output
     refute_stderr
 }
+
+@test "a local item is not tainted" {
+    # not from cscli... inspect
+    rune -0 mkdir -p "$CONFIG_DIR/collections"
+    rune -0 touch "$CONFIG_DIR/collections/foobar.yaml"
+    rune -0 cscli collections inspect foobar.yaml -o json
+    rune -0 jq -e '.tainted==false' <(output)
+
+    rune -0 cscli collections install crowdsecurity/sshd
+    rune -0 truncate -s0 "$CONFIG_DIR/collections/sshd.yaml"
+    rune -0 cscli collections inspect crowdsecurity/sshd -o json
+    rune -0 jq -e '.tainted==true' <(output)
+
+    # and not from hub update
+    rune -0 cscli hub update
+    assert_stderr --partial "collection crowdsecurity/sshd is tainted"
+    refute_stderr --partial "collection foobar.yaml is tainted"
+}
+
+@test "a local item's name defaults to its filename" {
+    rune -0 mkdir -p "$CONFIG_DIR/collections"
+    rune -0 touch "$CONFIG_DIR/collections/foobar.yaml"
+    rune -0 cscli collections list -o json
+    rune -0 jq -r '.[][].name' <(output)
+    assert_output "foobar.yaml"
+    rune -0 cscli collections list foobar.yaml
+    rune -0 cscli collections inspect foobar.yaml -o json
+    rune -0 jq -e '.installed==true' <(output)
+}
+
+@test "a local item can provide its own name" {
+    rune -0 mkdir -p "$CONFIG_DIR/collections"
+    echo "name: hi-its-me" > "$CONFIG_DIR/collections/foobar.yaml"
+    rune -0 cscli collections list -o json
+    rune -0 jq -r '.[][].name' <(output)
+    assert_output "hi-its-me"
+    rune -0 cscli collections list hi-its-me
+    rune -0 cscli collections inspect hi-its-me -o json
+    rune -0 jq -e '.installed==true' <(output)
+}