cscli setup: accept stdin; fix proftpd detection test and service unmask (#2496)

cmd/crowdsec-cli/setup.go

@@ -112,6 +112,20 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	var detectReader *os.File
+
+	switch detectConfigFile {
+	case "-":
+		log.Tracef("Reading detection rules from stdin")
+		detectReader = os.Stdin
+	default:
+		log.Tracef("Reading detection rules: %s", detectConfigFile)
+		detectReader, err = os.Open(detectConfigFile)
+		if err != nil {
+			return err
+		}
+	}
+
 	listSupportedServices, err := flags.GetBool("list-supported-services")
 	if err != nil {
 		return err
@@ -171,7 +185,7 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
 	}
 
 	if listSupportedServices {
-		supported, err := setup.ListSupported(detectConfigFile)
+		supported, err := setup.ListSupported(detectReader)
 		if err != nil {
 			return err
 		}
@@ -195,7 +209,7 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
 		SnubSystemd:  snubSystemd,
 	}
 
-	hubSetup, err := setup.Detect(detectConfigFile, opts)
+	hubSetup, err := setup.Detect(detectReader, opts)
 	if err != nil {
 		return fmt.Errorf("detecting services: %w", err)
 	}

pkg/setup/detect.go

@@ -3,6 +3,7 @@ package setup
 import (
 	"bytes"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"sort"
@@ -86,19 +87,19 @@ func validateDataSource(opaqueDS DataSourceItem) error {
 	return nil
 }
 
-func readDetectConfig(file string) (DetectConfig, error) {
+func readDetectConfig(fin io.Reader) (DetectConfig, error) {
 	var dc DetectConfig
 
-	yamlBytes, err := os.ReadFile(file)
+	yamlBytes, err := io.ReadAll(fin)
 	if err != nil {
-		return DetectConfig{}, fmt.Errorf("while reading file: %w", err)
+		return DetectConfig{}, err
 	}
 
 	dec := yaml.NewDecoder(bytes.NewBuffer(yamlBytes))
 	dec.KnownFields(true)
 
 	if err = dec.Decode(&dc); err != nil {
-		return DetectConfig{}, fmt.Errorf("while parsing %s: %w", file, err)
+		return DetectConfig{}, err
 	}
 
 	switch dc.Version {
@@ -107,7 +108,7 @@ func readDetectConfig(file string) (DetectConfig, error) {
 	case "1.0":
 		// all is well
 	default:
-		return DetectConfig{}, fmt.Errorf("unsupported version tag '%s' (must be 1.0)", dc.Version)
+		return DetectConfig{}, fmt.Errorf("invalid version tag '%s' (must be 1.0)", dc.Version)
 	}
 
 	for name, svc := range dc.Detect {
@@ -457,15 +458,13 @@ type DetectOptions struct {
 // Detect performs the service detection from a given configuration.
 // It outputs a setup file that can be used as input to "cscli setup install-hub"
 // or "cscli setup datasources".
-func Detect(serviceDetectionFile string, opts DetectOptions) (Setup, error) {
+func Detect(detectReader io.Reader, opts DetectOptions) (Setup, error) {
 	ret := Setup{}
 
 	// explicitly initialize to avoid json mashaling an empty slice as "null"
 	ret.Setup = make([]ServiceSetup, 0)
 
-	log.Tracef("Reading detection rules: %s", serviceDetectionFile)
+	sc, err := readDetectConfig(detectReader)
-
-	sc, err := readDetectConfig(serviceDetectionFile)
 	if err != nil {
 		return ret, err
 	}
@@ -559,8 +558,8 @@ func Detect(serviceDetectionFile string, opts DetectOptions) (Setup, error) {
 }
 
 // ListSupported parses the configuration file and outputs a list of the supported services.
-func ListSupported(serviceDetectionFile string) ([]string, error) {
+func ListSupported(detectConfig io.Reader) ([]string, error) {
-	dc, err := readDetectConfig(serviceDetectionFile)
+	dc, err := readDetectConfig(detectConfig)
 	if err != nil {
 		return nil, err
 	}

pkg/setup/detect_test.go

@@ -10,7 +10,6 @@ import (
 	"github.com/lithammer/dedent"
 	"github.com/stretchr/testify/require"
 
-	"github.com/crowdsecurity/go-cs-lib/csstring"
 	"github.com/crowdsecurity/go-cs-lib/cstest"
 
 	"github.com/crowdsecurity/crowdsec/pkg/setup"
@@ -58,7 +57,7 @@ func TestSetupHelperProcess(t *testing.T) {
 	os.Exit(0)
 }
 
-func tempYAML(t *testing.T, content string) string {
+func tempYAML(t *testing.T, content string) os.File {
 	t.Helper()
 	require := require.New(t)
 	file, err := os.CreateTemp("", "")
@@ -70,7 +69,10 @@ func tempYAML(t *testing.T, content string) string {
 	err = file.Close()
 	require.NoError(err)
 
-	return file.Name()
+	file, err = os.Open(file.Name())
+	require.NoError(err)
+
+	return *file
 }
 
 func TestPathExists(t *testing.T) {
@@ -239,7 +241,7 @@ func TestListSupported(t *testing.T) {
 			"invalid yaml: bad version",
 			"version: 2.0",
 			nil,
-			"unsupported version tag '2.0' (must be 1.0)",
+			"invalid version tag '2.0' (must be 1.0)",
 		},
 	}
 
@@ -248,8 +250,8 @@ func TestListSupported(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
 			f := tempYAML(t, tc.yml)
-			defer os.Remove(f)
+			defer os.Remove(f.Name())
-			supported, err := setup.ListSupported(f)
+			supported, err := setup.ListSupported(&f)
 			cstest.RequireErrorContains(t, err, tc.expectedErr)
 			require.ElementsMatch(t, tc.expected, supported)
 		})
@@ -373,9 +375,9 @@ func TestDetectSimpleRule(t *testing.T) {
 	      - false
 	  ugly:
 	`)
-	defer os.Remove(f)
+	defer os.Remove(f.Name())
 
-	detected, err := setup.Detect(f, setup.DetectOptions{})
+	detected, err := setup.Detect(&f, setup.DetectOptions{})
 	require.NoError(err)
 
 	expected := []setup.ServiceSetup{
@@ -420,9 +422,9 @@ detect:
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
 			f := tempYAML(t, tc.config)
-			defer os.Remove(f)
+			defer os.Remove(f.Name())
 
-			detected, err := setup.Detect(f, setup.DetectOptions{})
+			detected, err := setup.Detect(&f, setup.DetectOptions{})
 			cstest.RequireErrorContains(t, err, tc.expectedErr)
 			require.Equal(tc.expected, detected)
 		})
@@ -514,9 +516,9 @@ detect:
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
 			f := tempYAML(t, tc.config)
-			defer os.Remove(f)
+			defer os.Remove(f.Name())
 
-			detected, err := setup.Detect(f, setup.DetectOptions{})
+			detected, err := setup.Detect(&f, setup.DetectOptions{})
 			cstest.RequireErrorContains(t, err, tc.expectedErr)
 			require.Equal(tc.expected, detected)
 		})
@@ -542,9 +544,9 @@ func TestDetectForcedUnit(t *testing.T) {
 	      journalctl_filter:
 	        - _SYSTEMD_UNIT=crowdsec-setup-forced.service
 	`)
-	defer os.Remove(f)
+	defer os.Remove(f.Name())
 
-	detected, err := setup.Detect(f, setup.DetectOptions{ForcedUnits: []string{"crowdsec-setup-forced.service"}})
+	detected, err := setup.Detect(&f, setup.DetectOptions{ForcedUnits: []string{"crowdsec-setup-forced.service"}})
 	require.NoError(err)
 
 	expected := setup.Setup{
@@ -580,9 +582,9 @@ func TestDetectForcedProcess(t *testing.T) {
 	    when:
 	      - ProcessRunning("foobar")
 	`)
-	defer os.Remove(f)
+	defer os.Remove(f.Name())
 
-	detected, err := setup.Detect(f, setup.DetectOptions{ForcedProcesses: []string{"foobar"}})
+	detected, err := setup.Detect(&f, setup.DetectOptions{ForcedProcesses: []string{"foobar"}})
 	require.NoError(err)
 
 	expected := setup.Setup{
@@ -610,9 +612,9 @@ func TestDetectSkipService(t *testing.T) {
 	    when:
 	      - ProcessRunning("foobar")
 	`)
-	defer os.Remove(f)
+	defer os.Remove(f.Name())
 
-	detected, err := setup.Detect(f, setup.DetectOptions{ForcedProcesses: []string{"foobar"}, SkipServices: []string{"wizard"}})
+	detected, err := setup.Detect(&f, setup.DetectOptions{ForcedProcesses: []string{"foobar"}, SkipServices: []string{"wizard"}})
 	require.NoError(err)
 
 	expected := setup.Setup{[]setup.ServiceSetup{}}
@@ -826,9 +828,9 @@ func TestDetectForcedOS(t *testing.T) {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
 			f := tempYAML(t, tc.config)
-			defer os.Remove(f)
+			defer os.Remove(f.Name())
 
-			detected, err := setup.Detect(f, setup.DetectOptions{ForcedOS: tc.forced})
+			detected, err := setup.Detect(&f, setup.DetectOptions{ForcedOS: tc.forced})
 			cstest.RequireErrorContains(t, err, tc.expectedErr)
 			require.Equal(tc.expected, detected)
 		})
@@ -882,7 +884,7 @@ func TestDetectDatasourceValidation(t *testing.T) {
 				    datasource:
 				    source: file`,
 			expected:    setup.Setup{Setup: []setup.ServiceSetup{}},
-			expectedErr: "while parsing {{.DetectYaml}}: yaml: unmarshal errors:\n  line 6: field source not found in type setup.Service",
+			expectedErr: "yaml: unmarshal errors:\n  line 6: field source not found in type setup.Service",
 		}, {
 			name: "source is mismatched",
 			config: `
@@ -1001,18 +1003,10 @@ func TestDetectDatasourceValidation(t *testing.T) {
 	for _, tc := range tests {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
-			detectYaml := tempYAML(t, tc.config)
+			f := tempYAML(t, tc.config)
-			defer os.Remove(detectYaml)
+			defer os.Remove(f.Name())
-
+			detected, err := setup.Detect(&f, setup.DetectOptions{})
-			data := map[string]string{
+			cstest.RequireErrorContains(t, err, tc.expectedErr)
-				"DetectYaml": detectYaml,
-			}
-
-			expectedErr, err := csstring.Interpolate(tc.expectedErr, data)
-			require.NoError(err)
-
-			detected, err := setup.Detect(detectYaml, setup.DetectOptions{})
-			cstest.RequireErrorContains(t, err, expectedErr)
 			require.Equal(tc.expected, detected)
 		})
 	}

test/ansible/debug_tools.yml

@@ -14,5 +14,6 @@
           - zsh-autosuggestions
           - zsh-syntax-highlighting
           - zsh-theme-powerlevel9k
+          - silversearcher-ag
       when:
         - ansible_facts.os_family == "Debian"

test/ansible/vagrant/common

@@ -14,14 +14,14 @@ end
 Vagrant.configure('2') do |config|
   config.vm.define 'crowdsec'
 
-  if ARGV.any? { |arg| arg == 'up' || arg == 'provision' }
+  if ARGV.any? { |arg| arg == 'up' || arg == 'provision' } && !ARGV.include?('--no-provision')
     unless ENV['DB_BACKEND']
       $stderr.puts "\e[31mThe DB_BACKEND environment variable is not defined. Please set up the environment and try again.\e[0m"
       exit 1
     end
   end
 
-  config.vm.provision 'shell', path: 'bootstrap' if File.exists?('bootstrap')
+  config.vm.provision 'shell', path: 'bootstrap' if File.exist?('bootstrap')
   config.vm.synced_folder '.', '/vagrant', disabled: true
 
   config.vm.provider :libvirt do |libvirt|

test/ansible/vagrant/wizard/centos-8/Vagrantfile

@@ -10,4 +10,4 @@ Vagrant.configure('2') do |config|
 end
 
 common = '../common'
-load common if File.exists?(common)
+load common if File.exist?(common)

test/ansible/vagrant/wizard/common

@@ -21,7 +21,7 @@ Vagrant.configure('2') do |config|
     end
   end
 
-  config.vm.provision 'shell', path: 'bootstrap' if File.exists?('bootstrap')
+  config.vm.provision 'shell', path: 'bootstrap' if File.exist?('bootstrap')
   config.vm.synced_folder '.', '/vagrant', disabled: true
 
   config.vm.provider :libvirt do |libvirt|

test/ansible/vagrant/wizard/debian-10-buster/Vagrantfile

@@ -9,4 +9,4 @@ Vagrant.configure('2') do |config|
 end
 
 common = '../common'
-load common if File.exists?(common)
+load common if File.exist?(common)

test/ansible/vagrant/wizard/debian-11-bullseye/Vagrantfile

@@ -9,4 +9,4 @@ Vagrant.configure('2') do |config|
 end
 
 common = '../common'
-load common if File.exists?(common)
+load common if File.exist?(common)

test/ansible/vagrant/wizard/debian-12-bookworm/Vagrantfile

@@ -9,4 +9,4 @@ Vagrant.configure('2') do |config|
 end
 
 common = '../common'
-load common if File.exists?(common)
+load common if File.exist?(common)

test/ansible/vagrant/wizard/fedora-36/Vagrantfile

@@ -8,4 +8,4 @@ Vagrant.configure('2') do |config|
 end
 
 common = '../common'
-load common if File.exists?(common)
+load common if File.exist?(common)

test/ansible/vagrant/wizard/ubuntu-22.04-jammy/Vagrantfile

@@ -3,9 +3,9 @@
 Vagrant.configure('2') do |config|
   config.vm.box = 'generic/ubuntu2204'
   config.vm.provision "shell", inline: <<-SHELL
-    sudo apt install -y aptitude kitty-terminfo
+    sudo env DEBIAN_FRONTEND=noninteractive apt install -y aptitude kitty-terminfo
   SHELL
 end
 
 common = '../common'
-load common if File.exists?(common)
+load common if File.exist?(common)

test/ansible/vagrant/wizard/ubuntu-22.10-kinetic/Vagrantfile

@@ -8,4 +8,4 @@ Vagrant.configure('2') do |config|
 end
 
 common = '../common'
-load common if File.exists?(common)
+load common if File.exist?(common)

test/bats-detect/proftpd-deb.bats

@@ -10,7 +10,8 @@ setup_file() {
 
 teardown_file() {
     load "../lib/teardown_file.sh"
-    deb-remove proftpd
+    systemctl stop proftpd.service || :
+    deb-remove proftpd proftpd-core
 }
 
 setup() {
@@ -32,6 +33,7 @@ setup() {
 
 @test "proftpd: install" {
     run -0 deb-install proftpd
+    run -0 sudo systemctl unmask proftpd.service
     run -0 sudo systemctl enable proftpd.service
 }
 

test/bats/07_setup.bats

@@ -70,7 +70,11 @@ teardown() {
     assert_line --partial "--skip-service strings      ignore a service, don't recommend hub/datasources (can be repeated)"
 
     rune -1 cscli setup detect --detect-config /path/does/not/exist
-    assert_stderr --partial "detecting services: while reading file: open /path/does/not/exist: no such file or directory"
+    assert_stderr --partial "open /path/does/not/exist: no such file or directory"
+
+    # - is stdin
+    rune -1 cscli setup detect --detect-config - <<< "{}"
+    assert_stderr --partial "detecting services: missing version tag (must be 1.0)"
 
     # rm -f "${HUB_DIR}/detect.yaml"
 }
@@ -144,7 +148,7 @@ teardown() {
 	EOT
 
     rune -1 cscli setup detect --list-supported-services --detect-config "$tempfile"
-    assert_stderr --partial "while parsing ${tempfile}: yaml: unmarshal errors:"
+    assert_stderr --partial "yaml: unmarshal errors:"
 
     rm -f "$tempfile"
 }