From 25bb23d8b7c952d1ae246ef70a06e4ab80a30bc3 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 8 Jun 2023 15:08:51 +0200 Subject: [PATCH] minor refactor to pkg/types, cscli machines (#2270) * cleanup: separate ui and logic * trim some code from pkg/types --- cmd/crowdsec-cli/config_backup.go | 15 +++--- cmd/crowdsec-cli/config_restore.go | 17 ++++--- cmd/crowdsec-cli/copyfile.go | 73 ++++++++++++++++++++++++++++++ cmd/crowdsec-cli/machines.go | 35 +++++++------- cmd/crowdsec-cli/machines_table.go | 6 ++- cmd/crowdsec-cli/support.go | 11 ++++- cmd/crowdsec-cli/utils.go | 6 +-- pkg/types/utils.go | 72 ----------------------------- 8 files changed, 122 insertions(+), 113 deletions(-) create mode 100644 cmd/crowdsec-cli/copyfile.go diff --git a/cmd/crowdsec-cli/config_backup.go b/cmd/crowdsec-cli/config_backup.go index 30cf729fe..c6356f63f 100644 --- a/cmd/crowdsec-cli/config_backup.go +++ b/cmd/crowdsec-cli/config_backup.go @@ -9,7 +9,6 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) @@ -43,7 +42,7 @@ func backupConfigToDirectory(dirPath string) error { if csConfig.ConfigPaths.SimulationFilePath != "" { backupSimulation := filepath.Join(dirPath, "simulation.yaml") - if err = types.CopyFile(csConfig.ConfigPaths.SimulationFilePath, backupSimulation); err != nil { + if err = CopyFile(csConfig.ConfigPaths.SimulationFilePath, backupSimulation); err != nil { return errors.Wrapf(err, "failed copy %s to %s", csConfig.ConfigPaths.SimulationFilePath, backupSimulation) } @@ -56,7 +55,7 @@ func backupConfigToDirectory(dirPath string) error { */ if csConfig.Crowdsec != nil && csConfig.Crowdsec.AcquisitionFilePath != "" { backupAcquisition := filepath.Join(dirPath, "acquis.yaml") - if err = types.CopyFile(csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil { + if err = CopyFile(csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil { return fmt.Errorf("failed copy %s to %s : %s", csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition, err) } } @@ -78,7 +77,7 @@ func backupConfigToDirectory(dirPath string) error { return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir) } - if err = types.CopyFile(acquisFile, targetFname); err != nil { + if err = CopyFile(acquisFile, targetFname); err != nil { return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err) } @@ -88,7 +87,7 @@ func backupConfigToDirectory(dirPath string) error { if ConfigFilePath != "" { backupMain := fmt.Sprintf("%s/config.yaml", dirPath) - if err = types.CopyFile(ConfigFilePath, backupMain); err != nil { + if err = CopyFile(ConfigFilePath, backupMain); err != nil { return fmt.Errorf("failed copy %s to %s : %s", ConfigFilePath, backupMain, err) } @@ -97,7 +96,7 @@ func backupConfigToDirectory(dirPath string) error { if csConfig.API != nil && csConfig.API.Server != nil && csConfig.API.Server.OnlineClient != nil && csConfig.API.Server.OnlineClient.CredentialsFilePath != "" { backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath) - if err = types.CopyFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds); err != nil { + if err = CopyFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds); err != nil { return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds, err) } @@ -106,7 +105,7 @@ func backupConfigToDirectory(dirPath string) error { if csConfig.API != nil && csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" { backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath) - if err = types.CopyFile(csConfig.API.Client.CredentialsFilePath, backupLAPICreds); err != nil { + if err = CopyFile(csConfig.API.Client.CredentialsFilePath, backupLAPICreds); err != nil { return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Client.CredentialsFilePath, backupLAPICreds, err) } @@ -115,7 +114,7 @@ func backupConfigToDirectory(dirPath string) error { if csConfig.API != nil && csConfig.API.Server != nil && csConfig.API.Server.ProfilesPath != "" { backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath) - if err = types.CopyFile(csConfig.API.Server.ProfilesPath, backupProfiles); err != nil { + if err = CopyFile(csConfig.API.Server.ProfilesPath, backupProfiles); err != nil { return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Server.ProfilesPath, backupProfiles, err) } diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index 79d36d428..78e141215 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -13,7 +13,6 @@ import ( "gopkg.in/yaml.v2" "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) @@ -38,7 +37,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { backupMain := fmt.Sprintf("%s/config.yaml", dirPath) if _, err = os.Stat(backupMain); err == nil { if csConfig.ConfigPaths != nil && csConfig.ConfigPaths.ConfigDir != "" { - if err = types.CopyFile(backupMain, fmt.Sprintf("%s/config.yaml", csConfig.ConfigPaths.ConfigDir)); err != nil { + if err = CopyFile(backupMain, fmt.Sprintf("%s/config.yaml", csConfig.ConfigPaths.ConfigDir)); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupMain, csConfig.ConfigPaths.ConfigDir, err) } } @@ -51,21 +50,21 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath) if _, err = os.Stat(backupCAPICreds); err == nil { - if err = types.CopyFile(backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath); err != nil { + if err = CopyFile(backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath, err) } } backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath) if _, err = os.Stat(backupLAPICreds); err == nil { - if err = types.CopyFile(backupLAPICreds, csConfig.API.Client.CredentialsFilePath); err != nil { + if err = CopyFile(backupLAPICreds, csConfig.API.Client.CredentialsFilePath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupLAPICreds, csConfig.API.Client.CredentialsFilePath, err) } } backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath) if _, err = os.Stat(backupProfiles); err == nil { - if err = types.CopyFile(backupProfiles, csConfig.API.Server.ProfilesPath); err != nil { + if err = CopyFile(backupProfiles, csConfig.API.Server.ProfilesPath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupProfiles, csConfig.API.Server.ProfilesPath, err) } } @@ -106,7 +105,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { backupSimulation := fmt.Sprintf("%s/simulation.yaml", dirPath) if _, err = os.Stat(backupSimulation); err == nil { - if err = types.CopyFile(backupSimulation, csConfig.ConfigPaths.SimulationFilePath); err != nil { + if err = CopyFile(backupSimulation, csConfig.ConfigPaths.SimulationFilePath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupSimulation, csConfig.ConfigPaths.SimulationFilePath, err) } } @@ -123,7 +122,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { if _, err = os.Stat(backupAcquisition); err == nil { log.Debugf("restoring backup'ed %s", backupAcquisition) - if err = types.CopyFile(backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath); err != nil { + if err = CopyFile(backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath, err) } } @@ -139,7 +138,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { log.Debugf("restoring %s to %s", acquisFile, targetFname) - if err = types.CopyFile(acquisFile, targetFname); err != nil { + if err = CopyFile(acquisFile, targetFname); err != nil { return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err) } } @@ -160,7 +159,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir) } - if err = types.CopyFile(acquisFile, targetFname); err != nil { + if err = CopyFile(acquisFile, targetFname); err != nil { return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err) } diff --git a/cmd/crowdsec-cli/copyfile.go b/cmd/crowdsec-cli/copyfile.go new file mode 100644 index 000000000..4de6cd6e2 --- /dev/null +++ b/cmd/crowdsec-cli/copyfile.go @@ -0,0 +1,73 @@ +package main + +import ( + "fmt" + "io" + "os" + "path/filepath" + + log "github.com/sirupsen/logrus" +) + + +/*help to copy the file, ioutil doesn't offer the feature*/ + +func copyFileContents(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return + } + defer func() { + cerr := out.Close() + if err == nil { + err = cerr + } + }() + if _, err = io.Copy(out, in); err != nil { + return + } + err = out.Sync() + return +} + +/*copy the file, ioutile doesn't offer the feature*/ +func CopyFile(sourceSymLink, destinationFile string) (err error) { + sourceFile, err := filepath.EvalSymlinks(sourceSymLink) + if err != nil { + log.Infof("Not a symlink : %s", err) + sourceFile = sourceSymLink + } + + sourceFileStat, err := os.Stat(sourceFile) + if err != nil { + return + } + if !sourceFileStat.Mode().IsRegular() { + // cannot copy non-regular files (e.g., directories, + // symlinks, devices, etc.) + return fmt.Errorf("copyFile: non-regular source file %s (%q)", sourceFileStat.Name(), sourceFileStat.Mode().String()) + } + destinationFileStat, err := os.Stat(destinationFile) + if err != nil { + if !os.IsNotExist(err) { + return + } + } else { + if !(destinationFileStat.Mode().IsRegular()) { + return fmt.Errorf("copyFile: non-regular destination file %s (%q)", destinationFileStat.Name(), destinationFileStat.Mode().String()) + } + if os.SameFile(sourceFileStat, destinationFileStat) { + return + } + } + if err = os.Link(sourceFile, destinationFile); err != nil { + err = copyFileContents(sourceFile, destinationFile) + } + return +} + diff --git a/cmd/crowdsec-cli/machines.go b/cmd/crowdsec-cli/machines.go index 25bd5acec..215943102 100644 --- a/cmd/crowdsec-cli/machines.go +++ b/cmd/crowdsec-cli/machines.go @@ -12,7 +12,6 @@ import ( "time" "github.com/AlecAivazis/survey/v2" - "github.com/enescakir/emoji" "github.com/fatih/color" "github.com/go-openapi/strfmt" "github.com/google/uuid" @@ -85,22 +84,21 @@ func generateID(prefix string) (string, error) { return prefix + suffix, nil } -func displayLastHeartBeat(m *ent.Machine, fancy bool) string { - var hbDisplay string - - if m.LastHeartbeat != nil { - lastHeartBeat := time.Now().UTC().Sub(*m.LastHeartbeat) - hbDisplay = lastHeartBeat.Truncate(time.Second).String() - if fancy && lastHeartBeat > 2*time.Minute { - hbDisplay = fmt.Sprintf("%s %s", emoji.Warning.String(), lastHeartBeat.Truncate(time.Second).String()) - } - } else { - hbDisplay = "-" - if fancy { - hbDisplay = emoji.Warning.String() + " -" - } +// getLastHeartbeat returns the last heartbeat timestamp of a machine +// and a boolean indicating if the machine is considered active or not. +func getLastHeartbeat(m *ent.Machine) (string, bool) { + if m.LastHeartbeat == nil { + return "-", false } - return hbDisplay + + elapsed := time.Now().UTC().Sub(*m.LastHeartbeat) + + hb := elapsed.Truncate(time.Second).String() + if elapsed > 2*time.Minute { + return hb, false + } + + return hb, true } func getAgents(out io.Writer, dbClient *database.Client) error { @@ -130,9 +128,10 @@ func getAgents(out io.Writer, dbClient *database.Client) error { } else { validated = "false" } - err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, displayLastHeartBeat(m, false)}) + hb, _ := getLastHeartbeat(m) + err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb}) if err != nil { - return fmt.Errorf("failed to write raw output : %s", err) + return fmt.Errorf("failed to write raw output: %w", err) } } csvwriter.Flush() diff --git a/cmd/crowdsec-cli/machines_table.go b/cmd/crowdsec-cli/machines_table.go index cc15bb51b..e166fb785 100644 --- a/cmd/crowdsec-cli/machines_table.go +++ b/cmd/crowdsec-cli/machines_table.go @@ -24,7 +24,11 @@ func getAgentsTable(out io.Writer, machines []*ent.Machine) { validated = emoji.Prohibited.String() } - t.AddRow(m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, displayLastHeartBeat(m, true)) + hb, active := getLastHeartbeat(m) + if !active { + hb = emoji.Warning.String() + " " + hb + } + t.AddRow(m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb) } t.Render() diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go index 013abf4b2..66c1493a4 100644 --- a/cmd/crowdsec-cli/support.go +++ b/cmd/crowdsec-cli/support.go @@ -26,7 +26,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/fflag" "github.com/crowdsecurity/crowdsec/pkg/models" - "github.com/crowdsecurity/crowdsec/pkg/types" ) const ( @@ -48,6 +47,14 @@ const ( SUPPORT_CROWDSEC_PROFILE_PATH = "config/profiles.yaml" ) +// from https://github.com/acarl005/stripansi +var reStripAnsi = regexp.MustCompile("[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))") + +func stripAnsiString(str string) string { + // the byte version doesn't strip correctly + return reStripAnsi.ReplaceAllString(str, "") +} + func collectMetrics() ([]byte, []byte, error) { log.Info("Collecting prometheus metrics") err := csConfig.LoadPrometheus() @@ -400,7 +407,7 @@ cscli support dump -f /tmp/crowdsec-support.zip log.Errorf("Could not add zip entry for %s: %s", filename, err) continue } - fw.Write([]byte(types.StripAnsiString(string(data)))) + fw.Write([]byte(stripAnsiString(string(data)))) } err = zipWriter.Close() diff --git a/cmd/crowdsec-cli/utils.go b/cmd/crowdsec-cli/utils.go index e7a520172..f0e71c96d 100644 --- a/cmd/crowdsec-cli/utils.go +++ b/cmd/crowdsec-cli/utils.go @@ -594,7 +594,7 @@ func RestoreHub(dirPath string) error { log.Infof("Going to restore local/tainted [%s]", tfile.Name()) sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name()) destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name()) - if err = types.CopyFile(sourceFile, destinationFile); err != nil { + if err = CopyFile(sourceFile, destinationFile); err != nil { return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err) } log.Infof("restored %s to %s", sourceFile, destinationFile) @@ -603,7 +603,7 @@ func RestoreHub(dirPath string) error { log.Infof("Going to restore local/tainted [%s]", file.Name()) sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name()) destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name()) - if err = types.CopyFile(sourceFile, destinationFile); err != nil { + if err = CopyFile(sourceFile, destinationFile); err != nil { return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err) } log.Infof("restored %s to %s", sourceFile, destinationFile) @@ -653,7 +653,7 @@ func BackupHub(dirPath string) error { } clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate) tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName) - if err = types.CopyFile(v.LocalPath, tfile); err != nil { + if err = CopyFile(v.LocalPath, tfile); err != nil { return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err) } clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile) diff --git a/pkg/types/utils.go b/pkg/types/utils.go index 42d3ba189..8177717cd 100644 --- a/pkg/types/utils.go +++ b/pkg/types/utils.go @@ -5,10 +5,7 @@ import ( "bytes" "encoding/gob" "fmt" - "io" "os" - "path/filepath" - "regexp" "strconv" "strings" "time" @@ -105,67 +102,6 @@ func ParseDuration(d string) (time.Duration, error) { return duration, nil } -/*help to copy the file, ioutil doesn't offer the feature*/ - -func copyFileContents(src, dst string) (err error) { - in, err := os.Open(src) - if err != nil { - return - } - defer in.Close() - out, err := os.Create(dst) - if err != nil { - return - } - defer func() { - cerr := out.Close() - if err == nil { - err = cerr - } - }() - if _, err = io.Copy(out, in); err != nil { - return - } - err = out.Sync() - return -} - -/*copy the file, ioutile doesn't offer the feature*/ -func CopyFile(sourceSymLink, destinationFile string) (err error) { - sourceFile, err := filepath.EvalSymlinks(sourceSymLink) - if err != nil { - log.Infof("Not a symlink : %s", err) - sourceFile = sourceSymLink - } - - sourceFileStat, err := os.Stat(sourceFile) - if err != nil { - return - } - if !sourceFileStat.Mode().IsRegular() { - // cannot copy non-regular files (e.g., directories, - // symlinks, devices, etc.) - return fmt.Errorf("copyFile: non-regular source file %s (%q)", sourceFileStat.Name(), sourceFileStat.Mode().String()) - } - destinationFileStat, err := os.Stat(destinationFile) - if err != nil { - if !os.IsNotExist(err) { - return - } - } else { - if !(destinationFileStat.Mode().IsRegular()) { - return fmt.Errorf("copyFile: non-regular destination file %s (%q)", destinationFileStat.Name(), destinationFileStat.Mode().String()) - } - if os.SameFile(sourceFileStat, destinationFileStat) { - return - } - } - if err = os.Link(sourceFile, destinationFile); err != nil { - err = copyFileContents(sourceFile, destinationFile) - } - return -} - func UtcNow() time.Time { return time.Now().UTC() } @@ -183,11 +119,3 @@ func GetLineCountForFile(filepath string) int { } return lc } - -// from https://github.com/acarl005/stripansi -var reStripAnsi = regexp.MustCompile("[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))") - -func StripAnsiString(str string) string { - // the byte version doesn't strip correctly - return reStripAnsi.ReplaceAllString(str, "") -}