From a6c57e51b70e8f323f6407b306e6b3c01866147f Mon Sep 17 00:00:00 2001 From: Hylke Bons Date: Wed, 29 Jun 2016 14:44:15 +0100 Subject: [PATCH] git: Simplify progress parsing and support LFS --- SparkleShare/Common/BaseController.cs | 31 +-- SparkleShare/Common/StatusIconController.cs | 48 ++--- Sparkles/BaseFetcher.cs | 6 + Sparkles/BaseRepository.cs | 28 +-- Sparkles/Command.cs | 1 - Sparkles/Git/GitCommand.cs | 95 ++++++++- Sparkles/Git/GitFetcher.cs | 85 +------- Sparkles/Git/GitRepository.cs | 205 +++++--------------- 8 files changed, 203 insertions(+), 296 deletions(-) diff --git a/SparkleShare/Common/BaseController.cs b/SparkleShare/Common/BaseController.cs index 5d32405d..828b5924 100644 --- a/SparkleShare/Common/BaseController.cs +++ b/SparkleShare/Common/BaseController.cs @@ -67,11 +67,12 @@ namespace SparkleShare { public bool RepositoriesLoaded { get; private set; } public string FoldersPath { get; private set; } - public double ProgressPercentage = 0.0; - public double ProgressSpeedUp = 0.0; - public double ProgressSpeedDown = 0.0; - - + public double ProgressPercentage = 0.0; + public double ProgressSpeedUp = 0.0; + public double ProgressSpeedDown = 0.0; + public string ProgressInformation = ""; + + public event ShowSetupWindowEventHandler ShowSetupWindowEvent = delegate { }; public delegate void ShowSetupWindowEventHandler (PageType page_type); @@ -417,18 +418,20 @@ namespace SparkleShare { repo.SyncStatusChanged += delegate (SyncStatus status) { if (status == SyncStatus.Idle) { - ProgressPercentage = 0.0; - ProgressSpeedUp = 0.0; - ProgressSpeedDown = 0.0; + ProgressPercentage = 0.0; + ProgressSpeedUp = 0.0; + ProgressSpeedDown = 0.0; + ProgressInformation = ""; } UpdateState (); }; repo.ProgressChanged += delegate { - ProgressPercentage = 0.0; - ProgressSpeedUp = 0.0; - ProgressSpeedDown = 0.0; + ProgressPercentage = 0.0; + ProgressSpeedUp = 0.0; + ProgressSpeedDown = 0.0; + ProgressInformation = ""; double percentage = 0.0; int repo_count = 0; @@ -445,6 +448,9 @@ namespace SparkleShare { if (rep.Status == SyncStatus.SyncDown) ProgressSpeedDown += rep.ProgressSpeed; } + + if (repo_count == 1) + ProgressInformation = repo.ProgressInformation; if (repo_count > 0) ProgressPercentage = percentage / repo_count; @@ -460,8 +466,7 @@ namespace SparkleShare { }; repo.ConflictResolved += delegate { - AlertNotificationRaised ("Resolved a file collision", - "Local and server versions were kept."); + AlertNotificationRaised ("Resolved a file collision", "Local and server versions were kept."); }; AddRepository (repo); diff --git a/SparkleShare/Common/StatusIconController.cs b/SparkleShare/Common/StatusIconController.cs index 744d83f7..516ba6ea 100644 --- a/SparkleShare/Common/StatusIconController.cs +++ b/SparkleShare/Common/StatusIconController.cs @@ -133,38 +133,11 @@ namespace SparkleShare { public delegate void UpdateQuitItemEventHandler (bool quit_item_enabled); public IconState CurrentState = IconState.Idle; - public string StateText = "Welcome to SparkleShare!"; + public string StateText = "Welcome to SparkleShare!"; public ProjectInfo [] Projects = new ProjectInfo [0]; - public int ProgressPercentage { - get { - return (int) SparkleShare.Controller.ProgressPercentage; - } - } - - public string ProgressSpeed { - get { - string progress_speed = ""; - - if (SparkleShare.Controller.ProgressSpeedDown == 0.0 && SparkleShare.Controller.ProgressSpeedUp > 0.0) { - progress_speed = SparkleShare.Controller.ProgressSpeedUp.ToSize () + "/s "; - - } else if (SparkleShare.Controller.ProgressSpeedUp == 0.0 && SparkleShare.Controller.ProgressSpeedDown > 0.0) { - progress_speed = SparkleShare.Controller.ProgressSpeedDown.ToSize () + "/s "; - - } else if (SparkleShare.Controller.ProgressSpeedUp > 0.0 && - SparkleShare.Controller.ProgressSpeedDown > 0.0) { - - progress_speed = "Up: " + SparkleShare.Controller.ProgressSpeedUp.ToSize () + "/s " + - "Down: " + SparkleShare.Controller.ProgressSpeedDown.ToSize () + "/s"; - } - - return progress_speed; - } - } - public bool RecentEventsItemEnabled { get { return (SparkleShare.Controller.Repositories.Length > 0); @@ -242,8 +215,23 @@ namespace SparkleShare { StateText = "Receiving…"; } - if (ProgressPercentage > 0) - StateText += " " + ProgressPercentage + "% " + ProgressSpeed; + int progress_percentage = (int) SparkleShare.Controller.ProgressPercentage; + string progress_speed = ""; + + if (SparkleShare.Controller.ProgressSpeedUp > 0.0 && SparkleShare.Controller.ProgressSpeedDown > 0.0) { + progress_speed = "Up: " + SparkleShare.Controller.ProgressSpeedUp.ToSize () + "/s " + + "Down: " + SparkleShare.Controller.ProgressSpeedDown.ToSize () + "/s"; + } + + if (SparkleShare.Controller.ProgressSpeedUp > 0.0) + progress_speed = SparkleShare.Controller.ProgressSpeedUp.ToSize () + "/s "; + + if (SparkleShare.Controller.ProgressSpeedDown > 0.0) + progress_speed = SparkleShare.Controller.ProgressSpeedDown.ToSize () + "/s "; + + if (progress_percentage > 0) + StateText += string.Format (" {0}% {1} – {2}", + progress_percentage, progress_speed, SparkleShare.Controller.ProgressInformation); UpdateIconEvent (CurrentState); UpdateStatusItemEvent (StateText); diff --git a/Sparkles/BaseFetcher.cs b/Sparkles/BaseFetcher.cs index 85039885..5ddc3bd4 100644 --- a/Sparkles/BaseFetcher.cs +++ b/Sparkles/BaseFetcher.cs @@ -215,7 +215,13 @@ namespace Sparkles { } + DateTime progress_last_change = DateTime.Now; + protected void OnProgressChanged (double percentage, double speed, string information) { + // Only trigger the ProgressChanged event once per second + if (DateTime.Compare (this.progress_last_change, DateTime.Now.Subtract (new TimeSpan (0, 0, 0, 1))) >= 0) + return; + ProgressChanged (percentage, speed, information); } diff --git a/Sparkles/BaseRepository.cs b/Sparkles/BaseRepository.cs index e34b59a6..35d0c72e 100644 --- a/Sparkles/BaseRepository.cs +++ b/Sparkles/BaseRepository.cs @@ -65,7 +65,8 @@ namespace Sparkles { DiskSpaceExceeded, UnreadableFiles, NotFound, - IncompatibleClientServer + IncompatibleClientServer, + Unknown } @@ -112,8 +113,10 @@ namespace Sparkles { public SyncStatus Status { get; set; } public ErrorStatus Error { get; protected set; } public bool IsBuffering { get; set; } - public double ProgressPercentage { get; set; } - public double ProgressSpeed { get; set; } + + public double ProgressPercentage { get; private set; } + public double ProgressSpeed { get; private set; } + public string ProgressInformation { get; private set; } public DateTime LastSync { get { @@ -163,7 +166,6 @@ namespace Sparkles { Watcher watcher; TimeSpan poll_interval = PollInterval.Short; DateTime last_poll = DateTime.Now; - DateTime progress_last_change = DateTime.Now; Timers.Timer remote_timer = new Timers.Timer () { Interval = 5000 }; DisconnectReason last_disconnect_reason = DisconnectReason.None; @@ -365,21 +367,25 @@ namespace Sparkles { } - protected void OnProgressChanged (double progress_percentage, double progress_speed) + DateTime progress_last_change = DateTime.Now; + + protected void OnProgressChanged (double percentage, double speed, string information) { - if (progress_percentage < 1) + if (percentage < 1) return; // Only trigger the ProgressChanged event once per second if (DateTime.Compare (this.progress_last_change, DateTime.Now.Subtract (new TimeSpan (0, 0, 0, 1))) >= 0) return; - if (progress_percentage == 100.0) - progress_percentage = 99.0; + if (percentage == 100.0) + percentage = 99.0; - ProgressPercentage = progress_percentage; - ProgressSpeed = progress_speed; - this.progress_last_change = DateTime.Now; + progress_last_change = DateTime.Now; + + ProgressPercentage = percentage; + ProgressSpeed = speed; + ProgressInformation = information; ProgressChanged (); } diff --git a/Sparkles/Command.cs b/Sparkles/Command.cs index 6e7678e2..3bda6b3c 100644 --- a/Sparkles/Command.cs +++ b/Sparkles/Command.cs @@ -18,7 +18,6 @@ using System; using System.Diagnostics; using System.IO; -using System.Reflection; namespace Sparkles { diff --git a/Sparkles/Git/GitCommand.cs b/Sparkles/Git/GitCommand.cs index a8e28078..1e0ab146 100644 --- a/Sparkles/Git/GitCommand.cs +++ b/Sparkles/Git/GitCommand.cs @@ -14,7 +14,8 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . using System; - +using System.Globalization; +using System.Text.RegularExpressions; namespace Sparkles.Git { @@ -95,6 +96,98 @@ namespace Sparkles.Git { } + static Regex progress_regex = new Regex (@"([0-9]+)%", RegexOptions.Compiled); + static Regex progress_regex_lfs = new Regex (@".*([0-9]+) of ([0-9]+).*", RegexOptions.Compiled); + static Regex speed_regex = new Regex (@"([0-9\.]+) ([KM])iB/s", RegexOptions.Compiled); + + public static ErrorStatus ParseProgress (string line, out double percentage, out double speed, out string information) + { + percentage = 0; + speed = 0; + information = ""; + + Match match; + + if (line.StartsWith ("Git LFS:")) { + match = progress_regex_lfs.Match (line); + + if (!match.Success) + return ErrorStatus.None; + + percentage = double.Parse (match.Groups [1].Value) / double.Parse (match.Groups [2].Value) * 100; + information = string.Format ("{0} of {1} files", match.Groups [1].Value, match.Groups [2].Value); + + return ErrorStatus.None; + } + + match = progress_regex.Match (line); + + if (!match.Success && !string.IsNullOrWhiteSpace (line)) { + Logger.LogInfo ("Git", line); + return FindError (line); + } + + int number = int.Parse (match.Groups [1].Value); + + // The pushing progress consists of two stages: the "Compressing + // objects" stage which we count as 20% of the total progress, and + // the "Writing objects" stage which we count as the last 80% + if (line.Contains ("Compressing objects")) { + // "Compressing objects" stage + percentage = (number / 100 * 20); + + } else if (line.Contains ("Writing objects")) { + percentage = (number / 100 * 80 + 20); + Match speed_match = speed_regex.Match (line); + + if (speed_match.Success) { + speed = double.Parse (speed_match.Groups [1].Value, new CultureInfo ("en-US")) * 1024; + + if (speed_match.Groups [2].Value.Equals ("M")) + speed = speed * 1024; + + information = speed.ToSize (); + } + } + + return ErrorStatus.None; + } + + + static ErrorStatus FindError (string line) + { + ErrorStatus error = ErrorStatus.None; + + if (line.Contains ("WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!") || + line.Contains ("WARNING: POSSIBLE DNS SPOOFING DETECTED!")) { + + error = ErrorStatus.HostIdentityChanged; + } + + if (line.StartsWith ("Permission denied") || + line.StartsWith ("ssh_exchange_identification: Connection closed by remote host") || + line.StartsWith ("The authenticity of host")) { + + error = ErrorStatus.AuthenticationFailed; + } + + if (line.EndsWith ("does not appear to be a git repository")) + error = ErrorStatus.NotFound; + + if (line.EndsWith ("expected old/new/ref, got 'shallow")) + error = ErrorStatus.IncompatibleClientServer; + + if (line.StartsWith ("error: Disk space exceeded") || + line.EndsWith ("No space left on device") || + line.EndsWith ("file write error (Disk quota exceeded)")) { + + error = ErrorStatus.DiskSpaceExceeded; + } + + return error; + } + + public static string FormatGitSSHCommand (SSHAuthenticationInfo auth_info) { return SSHPath + " " + diff --git a/Sparkles/Git/GitFetcher.cs b/Sparkles/Git/GitFetcher.cs index e27a471c..60d592cf 100644 --- a/Sparkles/Git/GitFetcher.cs +++ b/Sparkles/Git/GitFetcher.cs @@ -106,9 +106,6 @@ namespace Sparkles.Git { if (FetchedRepoStorageType == StorageType.LargeFiles) output_stream = git_clone.StandardOutput; - var last_change = DateTime.Now; - var change_interval = new TimeSpan (0, 0, 0, 1); - double previous_percentage = 0; double percentage = 0; double speed = 0; @@ -118,9 +115,9 @@ namespace Sparkles.Git { string line = output_stream.ReadLine (); previous_percentage = percentage; - bool parse_success = ParseProgress (line, out percentage, out speed, out information); + ErrorStatus error = GitCommand.ParseProgress (line, out percentage, out speed, out information); - if (!parse_success) { + if (error != ErrorStatus.None) { IsActive = false; git_clone.Kill (); git_clone.Dispose (); @@ -130,12 +127,6 @@ namespace Sparkles.Git { if (percentage <= previous_percentage) continue; - - if (DateTime.Compare (last_change, DateTime.Now.Subtract (change_interval)) < 0) { - Console.WriteLine (percentage); - OnProgressChanged (percentage, speed, information); - last_change = DateTime.Now; - } } git_clone.WaitForExit (); @@ -151,78 +142,6 @@ namespace Sparkles.Git { } - Regex progress_regex = new Regex (@"([0-9]+)%", RegexOptions.Compiled); - Regex progress_regex_lfs = new Regex (@"^Git LFS:.*([0-9]+) of ([0-9]+).*", RegexOptions.Compiled); - Regex speed_regex = new Regex (@"([0-9\.]+) ([KM])iB/s", RegexOptions.Compiled); - - public bool ParseProgress (string line, out double percentage, out double speed, out string information) - { - percentage = 0; - speed = 0; - information = ""; - - Match match; - - if (FetchedRepoStorageType == StorageType.LargeFiles) { - match = progress_regex_lfs.Match (line); - - if (!match.Success) - return true; - - percentage = double.Parse (match.Groups [1].Value) / double.Parse (match.Groups [2].Value) * 100; - information = string.Format ("{0} of {1} files", match.Groups [1].Value, match.Groups [2].Value); - - return true; - } - - match = progress_regex.Match (line); - - if (!match.Success) { - Logger.LogInfo ("Fetcher", line); - line = line.Trim (new char [] { ' ', '@' }); - - if (line.StartsWith ("fatal:", StringComparison.InvariantCultureIgnoreCase) || - line.StartsWith ("error:", StringComparison.InvariantCultureIgnoreCase)) { - - errors.Add (line); - - } else if (line.StartsWith ("WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!")) { - errors.Add ("warning: Remote host identification has changed"); - - } else if (line.StartsWith ("WARNING: POSSIBLE DNS SPOOFING DETECTED!")) { - errors.Add ("warning: Possible DNS spoofing detected"); - } - - return false; - } - - int number = int.Parse (match.Groups [1].Value, new CultureInfo ("en-US")); - - // The pushing progress consists of two stages: the "Compressing - // objects" stage which we count as 20% of the total progress, and - // the "Writing objects" stage which we count as the last 80% - if (line.Contains ("Compressing objects")) { - // "Compressing objects" stage - percentage = (number / 100 * 20); - - } else if (line.Contains ("Writing objects")) { - percentage = (number / 100 * 80 + 20); - Match speed_match = speed_regex.Match (line); - - if (speed_match.Success) { - speed = double.Parse (speed_match.Groups [1].Value, new CultureInfo ("en-US")) * 1024; - - if (speed_match.Groups [2].Value.Equals ("M")) - speed = speed * 1024; - - information = speed.ToSize (); - } - } - - return true; - } - - public override void Stop () { try { diff --git a/Sparkles/Git/GitRepository.cs b/Sparkles/Git/GitRepository.cs index 568c093c..055a86ee 100644 --- a/Sparkles/Git/GitRepository.cs +++ b/Sparkles/Git/GitRepository.cs @@ -17,7 +17,6 @@ using System; using System.Collections.Generic; -using System.Globalization; using System.IO; using System.Text; using System.Text.RegularExpressions; @@ -224,61 +223,8 @@ namespace Sparkles.Git { git_push.StartInfo.RedirectStandardError = true; git_push.Start (); - double percentage = 1.0; - - // TODO: parse LFS progress - while (!git_push.StandardError.EndOfStream) { - string line = git_push.StandardError.ReadLine (); - Console.WriteLine (line); - Match match = this.progress_regex.Match (line); - double speed = 0.0; - double number = 0.0; - - if (match.Success) { - try { - number = double.Parse (match.Groups [1].Value, new CultureInfo ("en-US")); - - } catch (FormatException) { - Logger.LogInfo ("Git", "Error parsing progress: \"" + match.Groups [1] + "\""); - } - - // The pushing progress consists of two stages: the "Compressing - // objects" stage which we count as 20% of the total progress, and - // the "Writing objects" stage which we count as the last 80% - if (line.StartsWith ("Compressing")) { - // "Compressing objects" stage - number = (number / 100 * 20); - - } else { - // "Writing objects" stage - number = (number / 100 * 80 + 20); - Match speed_match = this.speed_regex.Match (line); - - if (speed_match.Success) { - try { - speed = double.Parse (speed_match.Groups [1].Value, new CultureInfo ("en-US")) * 1024; - - } catch (FormatException) { - Logger.LogInfo ("Git", "Error parsing speed: \"" + speed_match.Groups [1] + "\""); - } - - if (speed_match.Groups [2].Value.Equals ("M")) - speed = speed * 1024; - } - } - - } else { - Logger.LogInfo ("Git", Name + " | " + line); - - if (FindError (line)) - return false; - } - - if (number >= percentage) { - percentage = number; - base.OnProgressChanged (percentage, speed); - } - } + if (!ReadStream (git_push)) + return false; git_push.WaitForExit (); @@ -299,70 +245,17 @@ namespace Sparkles.Git { if (StorageType == StorageType.LargeFiles) File.Create (lfs_is_behind_file_path); - var git = new GitCommand (LocalPath, "fetch --progress origin " + branch, auth_info); + var git_fetch = new GitCommand (LocalPath, "fetch --progress origin " + branch, auth_info); - git.StartInfo.RedirectStandardError = true; - git.Start (); + git_fetch.StartInfo.RedirectStandardError = true; + git_fetch.Start (); - double percentage = 1.0; + if (!ReadStream (git_fetch)) + return false; - // TODO: parse LFS progress - while (!git.StandardError.EndOfStream) { - string line = git.StandardError.ReadLine (); - Match match = this.progress_regex.Match (line); - double speed = 0.0; - double number = 0.0; + git_fetch.WaitForExit (); - if (match.Success) { - try { - number = double.Parse (match.Groups [1].Value, new CultureInfo ("en-US")); - - } catch (FormatException) { - Logger.LogInfo ("Git", "Error parsing progress: \"" + match.Groups [1] + "\""); - } - - // The fetching progress consists of two stages: the "Compressing - // objects" stage which we count as 20% of the total progress, and - // the "Receiving objects" stage which we count as the last 80% - if (line.StartsWith ("Compressing")) { - // "Compressing objects" stage - number = (number / 100 * 20); - - } else { - // "Writing objects" stage - number = (number / 100 * 80 + 20); - Match speed_match = this.speed_regex.Match (line); - - if (speed_match.Success) { - try { - speed = double.Parse (speed_match.Groups [1].Value, new CultureInfo ("en-US")) * 1024; - - } catch (FormatException) { - Logger.LogInfo ("Git", "Error parsing speed: \"" + speed_match.Groups [1] + "\""); - } - - if (speed_match.Groups [2].Value.Equals ("M")) - speed = speed * 1024; - } - } - - } else { - Logger.LogInfo ("Git", Name + " | " + line); - - if (FindError (line)) - return false; - } - - - if (number >= percentage) { - percentage = number; - base.OnProgressChanged (percentage, speed); - } - } - - git.WaitForExit (); - - if (git.ExitCode != 0) { + if (git_fetch.ExitCode != 0) { Error = ErrorStatus.HostUnreachable; return false; } @@ -390,6 +283,45 @@ namespace Sparkles.Git { } + bool ReadStream (GitCommand command) + { + StreamReader output_stream = command.StandardError; + + if (StorageType == StorageType.LargeFiles) + output_stream = command.StandardOutput; + + double previous_percentage = 0; + double percentage = 0; + double speed = 0; + string information = ""; + + while (!output_stream.EndOfStream) { + string line = output_stream.ReadLine (); + + previous_percentage = percentage; + ErrorStatus error = GitCommand.ParseProgress (line, out percentage, out speed, out information); + + if (error != ErrorStatus.None) { + Error = error; + information = line; + + command.Kill (); + command.Dispose (); + Logger.LogInfo ("Git", Name + " | Error status changed to " + Error); + + return false; + } + + if (percentage <= previous_percentage) + continue; + + OnProgressChanged (percentage, speed, information); + } + + return true; + } + + public override bool HasLocalChanges { get { PrepareDirectories (LocalPath); @@ -685,44 +617,6 @@ namespace Sparkles.Git { } - bool FindError (string line) - { - Error = ErrorStatus.None; - - if (line.Contains ("WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!") || - line.Contains ("WARNING: POSSIBLE DNS SPOOFING DETECTED!")) { - - Error = ErrorStatus.HostIdentityChanged; - - } else if (line.StartsWith ("Permission denied") || - line.StartsWith ("ssh_exchange_identification: Connection closed by remote host") || - line.StartsWith ("The authenticity of host")) { - - Error = ErrorStatus.AuthenticationFailed; - - } else if (line.EndsWith ("does not appear to be a git repository")) { - Error = ErrorStatus.NotFound; - - } else if (line.EndsWith ("expected old/new/ref, got 'shallow")) { - Error = ErrorStatus.IncompatibleClientServer; - - } else if (line.StartsWith ("error: Disk space exceeded") || - line.EndsWith ("No space left on device") || - line.EndsWith ("file write error (Disk quota exceeded)")) { - - Error = ErrorStatus.DiskSpaceExceeded; - } - - if (Error != ErrorStatus.None) { - Logger.LogInfo ("Git", Name + " | Error status changed to " + Error); - return true; - - } else { - return false; - } - } - - public override List UnsyncedChanges { get { return ParseStatus (); @@ -1176,9 +1070,6 @@ namespace Sparkles.Git { } - Regex progress_regex = new Regex (@"([0-9]+)%", RegexOptions.Compiled); - Regex speed_regex = new Regex (@"([0-9\.]+) ([KM])iB/s", RegexOptions.Compiled); - Regex log_regex = new Regex (@"commit ([a-f0-9]{40})*\n" + "Author: (.+) <(.+)>\n" + "Date: ([0-9]{4})-([0-9]{2})-([0-9]{2}) " +