From 8ab758ccda1c6f623829432179110ee1182eb8d8 Mon Sep 17 00:00:00 2001 From: Hylke Bons Date: Tue, 28 Jun 2016 09:41:17 +0100 Subject: [PATCH] git fetcher: Implement LFS progress reporting --- SparkleShare/Common/BaseController.cs | 6 +- SparkleShare/Common/SetupController.cs | 12 +- SparkleShare/Mac/UserInterface/Setup.cs | 57 ++++++ Sparkles/BaseFetcher.cs | 6 +- Sparkles/Git/GitFetcher.cs | 242 +++++++++++++----------- Sparkles/Git/GitRepository.cs | 1 + 6 files changed, 201 insertions(+), 123 deletions(-) diff --git a/SparkleShare/Common/BaseController.cs b/SparkleShare/Common/BaseController.cs index 7382a2a5..b438ee38 100644 --- a/SparkleShare/Common/BaseController.cs +++ b/SparkleShare/Common/BaseController.cs @@ -88,7 +88,7 @@ namespace SparkleShare { public delegate void FolderFetchErrorHandler (string remote_url, string [] errors); public event FolderFetchingHandler FolderFetching = delegate { }; - public delegate void FolderFetchingHandler (double percentage, double speed); + public delegate void FolderFetchingHandler (double percentage, double speed, string information); public event Action FolderListChanged = delegate { }; @@ -611,9 +611,9 @@ namespace SparkleShare { } - void FetcherProgressChangedDelgate (double percentage, double speed) + void FetcherProgressChangedDelgate (double percentage, double speed, string information) { - FolderFetching (percentage, speed); + FolderFetching (percentage, speed, information); } diff --git a/SparkleShare/Common/SetupController.cs b/SparkleShare/Common/SetupController.cs index ad6a2b6e..b3541989 100644 --- a/SparkleShare/Common/SetupController.cs +++ b/SparkleShare/Common/SetupController.cs @@ -53,7 +53,7 @@ namespace SparkleShare { public delegate void ChangePageEventHandler (PageType page, string [] warnings); public event UpdateProgressBarEventHandler UpdateProgressBarEvent = delegate { }; - public delegate void UpdateProgressBarEventHandler (double percentage, string speed); + public delegate void UpdateProgressBarEventHandler (double percentage, string information); public event UpdateSetupContinueButtonEventHandler UpdateSetupContinueButtonEvent = delegate { }; public delegate void UpdateSetupContinueButtonEventHandler (bool button_enabled); @@ -370,14 +370,14 @@ namespace SparkleShare { SparkleShare.Controller.FolderFetching -= SyncingPageFetchingDelegate; } - private void SyncingPageFetchingDelegate (double percentage, double speed) + private void SyncingPageFetchingDelegate (double percentage, double speed ,string information) { ProgressBarPercentage = percentage; - if (speed == 0.0) - UpdateProgressBarEvent (ProgressBarPercentage, ""); - else - UpdateProgressBarEvent (ProgressBarPercentage, "Fetching files… " + speed.ToSize () + "/s"); + if (speed > 0) + information = speed.ToSize () + " – " + information; + + UpdateProgressBarEvent (ProgressBarPercentage, information); } diff --git a/SparkleShare/Mac/UserInterface/Setup.cs b/SparkleShare/Mac/UserInterface/Setup.cs index f5cbe0f1..e4eb3f90 100644 --- a/SparkleShare/Mac/UserInterface/Setup.cs +++ b/SparkleShare/Mac/UserInterface/Setup.cs @@ -51,6 +51,12 @@ namespace SparkleShare { private SparkleDataSource DataSource; + + private NSButtonCell ButtonCellProto; + + private NSMatrix Matrix; + + public Setup () : base () { Controller.HideWindowEvent += delegate { @@ -456,6 +462,57 @@ namespace SparkleShare { Buttons.Add (CancelButton); } + + + if (type == PageType.StorageSetup) { + Header = string.Format ("Storage type for ‘{0}’", Controller.SyncingFolder); + Description = "What type of storage would you like to use?"; + + + ButtonCellProto = new NSButtonCell (); + ButtonCellProto.SetButtonType (NSButtonType.Radio); + ButtonCellProto.Font = NSFont.FromFontName (UserInterface.FontName + " Bold", NSFont.SystemFontSize); + + + Matrix = new NSMatrix (new RectangleF (215, 0, 256, 256), NSMatrixMode.Radio, + ButtonCellProto, SparkleShare.Controller.FetcherAvailableStorageTypes.Count, 1); + Matrix.BackgroundColor = NSColor.Yellow; + Matrix.CellSize = new SizeF (256, 18); + Matrix.IntercellSpacing = new SizeF (12, 12); + + int i = 0; + foreach (StorageTypeInfo storage_type in SparkleShare.Controller.FetcherAvailableStorageTypes) { + Matrix.Cells [i].Title = storage_type.Name; + + // Matrix.IntercellSpacing = new SizeF (30, 30); + // todo: description + i++; + } + + ContentView.AddSubview (Matrix); + + + CancelButton = new NSButton () { Title = "Cancel" }; + ContinueButton = new NSButton () { Title = "Continue" }; + + + ContinueButton.Activated += delegate { + Console.WriteLine (Matrix.SelectedRow); + + StorageTypeInfo selected_storage_type = SparkleShare.Controller.FetcherAvailableStorageTypes [Matrix.SelectedRow]; + Controller.StoragePageCompleted (selected_storage_type.Type); + }; + + CancelButton.Activated += delegate { Controller.SyncingCancelled (); }; + + + Buttons.Add (ContinueButton); + Buttons.Add (CancelButton); + + MakeFirstResponder ((NSResponder)PasswordTextField); + NSApplication.SharedApplication.RequestUserAttention (NSRequestUserAttentionType.CriticalRequest); + } + if (type == PageType.CryptoSetup || type == PageType.CryptoPassword) { if (type == PageType.CryptoSetup) { Header = "Set up file encryption"; diff --git a/Sparkles/BaseFetcher.cs b/Sparkles/BaseFetcher.cs index c2fd5317..85039885 100644 --- a/Sparkles/BaseFetcher.cs +++ b/Sparkles/BaseFetcher.cs @@ -45,7 +45,7 @@ namespace Sparkles { public delegate void FinishedEventHandler (StorageType storage_type, string [] warnings); public event ProgressChangedEventHandler ProgressChanged = delegate { }; - public delegate void ProgressChangedEventHandler (double percentage, double speed); + public delegate void ProgressChangedEventHandler (double percentage, double speed, string information); public abstract bool Fetch (); @@ -215,8 +215,8 @@ namespace Sparkles { } - protected void OnProgressChanged (double percentage, double speed) { - ProgressChanged (percentage, speed); + protected void OnProgressChanged (double percentage, double speed, string information) { + ProgressChanged (percentage, speed, information); } diff --git a/Sparkles/Git/GitFetcher.cs b/Sparkles/Git/GitFetcher.cs index 904bc3b9..e27a471c 100644 --- a/Sparkles/Git/GitFetcher.cs +++ b/Sparkles/Git/GitFetcher.cs @@ -25,15 +25,11 @@ namespace Sparkles.Git { public class GitFetcher : SSHFetcher { - SSHAuthenticationInfo auth_info; GitCommand git_clone; + SSHAuthenticationInfo auth_info; string password_salt = Path.GetRandomFileName ().SHA256 ().Substring (0, 16); - Regex progress_regex = new Regex (@"([0-9]+)%", RegexOptions.Compiled); - Regex speed_regex = new Regex (@"([0-9\.]+) ([KM])iB/s", RegexOptions.Compiled); - - protected override bool IsFetchedRepoEmpty { get { @@ -78,38 +74,6 @@ namespace Sparkles.Git { } - StorageType? DetermineStorageType () - { - var git_ls_remote = new GitCommand (Configuration.DefaultConfiguration.TmpPath, - string.Format ("ls-remote --heads \"{0}\"", RemoteUrl), auth_info); - - string output = git_ls_remote.StartAndReadStandardOutput (); - - if (git_ls_remote.ExitCode != 0) - return null; - - if (string.IsNullOrWhiteSpace (output)) - return StorageType.Unknown; - - foreach (string line in output.Split ("\n".ToCharArray ())) { - string [] line_parts = line.Split ('/'); - string branch = line_parts [line_parts.Length - 1]; - - if (branch == "x-sparkleshare-lfs") - return StorageType.LargeFiles; - - string encrypted_storage_prefix = "x-sparkleshare-encrypted-"; - - if (branch.StartsWith (encrypted_storage_prefix)) { - password_salt = branch.Replace (encrypted_storage_prefix, ""); - return StorageType.Encrypted; - } - } - - return StorageType.Plain; - } - - public override bool Fetch () { if (!base.Fetch ()) @@ -130,88 +94,48 @@ namespace Sparkles.Git { if (storage_type == StorageType.LargeFiles) git_clone_command = "lfs clone --progress --no-checkout"; - var git_clone = new GitCommand (Configuration.DefaultConfiguration.TmpPath, + git_clone = new GitCommand (Configuration.DefaultConfiguration.TmpPath, string.Format ("{0} \"{1}\" \"{2}\"", git_clone_command, RemoteUrl, TargetFolder), auth_info); git_clone.StartInfo.RedirectStandardError = true; git_clone.Start (); - double percentage = 1.0; + StreamReader output_stream = git_clone.StandardError; + + if (FetchedRepoStorageType == StorageType.LargeFiles) + output_stream = git_clone.StandardOutput; var last_change = DateTime.Now; - var change_interval = new TimeSpan (0, 0, 0, 1); + var change_interval = new TimeSpan (0, 0, 0, 1); - try { - while (!git_clone.StandardError.EndOfStream) { - string line = git_clone.StandardError.ReadLine (); - Match match = progress_regex.Match (line); + double previous_percentage = 0; + double percentage = 0; + double speed = 0; + string information = ""; - double number = 0.0; - double speed = 0.0; - if (match.Success) { - try { - number = double.Parse (match.Groups [1].Value, new CultureInfo ("en-US")); + while (!output_stream.EndOfStream) { + string line = output_stream.ReadLine (); - } catch (FormatException) { - Logger.LogInfo ("Git", "Error parsing progress: \"" + match.Groups [1] + "\""); - } + previous_percentage = percentage; + bool parse_success = ParseProgress (line, out percentage, out speed, out information); - // The pushing progress consists of two stages: the "Compressing - // objects" stage which we count as 20% of the total progress, and - // the "Writing objects" stage which we count as the last 80% - if (line.Contains ("Compressing")) { - // "Compressing objects" stage - number = (number / 100 * 20); + if (!parse_success) { + IsActive = false; + git_clone.Kill (); + git_clone.Dispose (); - } else { - // "Writing objects" stage - number = (number / 100 * 80 + 20); - Match speed_match = speed_regex.Match (line); - - if (speed_match.Success) { - try { - speed = double.Parse (speed_match.Groups [1].Value, new CultureInfo ("en-US")) * 1024; - - } catch (FormatException) { - Logger.LogInfo ("Git", "Error parsing speed: \"" + speed_match.Groups [1] + "\""); - } - - if (speed_match.Groups [2].Value.Equals ("M")) - speed = speed * 1024; - } - } - - } else { - Logger.LogInfo ("Fetcher", line); - line = line.Trim (new char [] {' ', '@'}); - - if (line.StartsWith ("fatal:", StringComparison.InvariantCultureIgnoreCase) || - line.StartsWith ("error:", StringComparison.InvariantCultureIgnoreCase)) { - - errors.Add (line); - - } else if (line.StartsWith ("WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!")) { - errors.Add ("warning: Remote host identification has changed!"); - - } else if (line.StartsWith ("WARNING: POSSIBLE DNS SPOOFING DETECTED!")) { - errors.Add ("warning: Possible DNS spoofing detected!"); - } - } - - if (number >= percentage) { - percentage = number; - - if (DateTime.Compare (last_change, DateTime.Now.Subtract (change_interval)) < 0) { - OnProgressChanged (percentage, speed); - last_change = DateTime.Now; - } - } + return false; } - } catch (Exception) { - IsActive = false; - return false; + if (percentage <= previous_percentage) + continue; + + if (DateTime.Compare (last_change, DateTime.Now.Subtract (change_interval)) < 0) { + Console.WriteLine (percentage); + OnProgressChanged (percentage, speed, information); + last_change = DateTime.Now; + } } git_clone.WaitForExit (); @@ -219,17 +143,81 @@ namespace Sparkles.Git { if (git_clone.ExitCode != 0) return false; - while (percentage < 100) { - percentage += 25; + Thread.Sleep (500); + OnProgressChanged (100, 0, ""); + Thread.Sleep (500); - if (percentage >= 100) - break; + return true; + } - Thread.Sleep (500); - OnProgressChanged (percentage, 0); + + Regex progress_regex = new Regex (@"([0-9]+)%", RegexOptions.Compiled); + Regex progress_regex_lfs = new Regex (@"^Git LFS:.*([0-9]+) of ([0-9]+).*", RegexOptions.Compiled); + Regex speed_regex = new Regex (@"([0-9\.]+) ([KM])iB/s", RegexOptions.Compiled); + + public bool ParseProgress (string line, out double percentage, out double speed, out string information) + { + percentage = 0; + speed = 0; + information = ""; + + Match match; + + if (FetchedRepoStorageType == StorageType.LargeFiles) { + match = progress_regex_lfs.Match (line); + + if (!match.Success) + return true; + + percentage = double.Parse (match.Groups [1].Value) / double.Parse (match.Groups [2].Value) * 100; + information = string.Format ("{0} of {1} files", match.Groups [1].Value, match.Groups [2].Value); + + return true; } - OnProgressChanged (100, 0); + match = progress_regex.Match (line); + + if (!match.Success) { + Logger.LogInfo ("Fetcher", line); + line = line.Trim (new char [] { ' ', '@' }); + + if (line.StartsWith ("fatal:", StringComparison.InvariantCultureIgnoreCase) || + line.StartsWith ("error:", StringComparison.InvariantCultureIgnoreCase)) { + + errors.Add (line); + + } else if (line.StartsWith ("WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!")) { + errors.Add ("warning: Remote host identification has changed"); + + } else if (line.StartsWith ("WARNING: POSSIBLE DNS SPOOFING DETECTED!")) { + errors.Add ("warning: Possible DNS spoofing detected"); + } + + return false; + } + + int number = int.Parse (match.Groups [1].Value, new CultureInfo ("en-US")); + + // The pushing progress consists of two stages: the "Compressing + // objects" stage which we count as 20% of the total progress, and + // the "Writing objects" stage which we count as the last 80% + if (line.Contains ("Compressing objects")) { + // "Compressing objects" stage + percentage = (number / 100 * 20); + + } else if (line.Contains ("Writing objects")) { + percentage = (number / 100 * 80 + 20); + Match speed_match = speed_regex.Match (line); + + if (speed_match.Success) { + speed = double.Parse (speed_match.Groups [1].Value, new CultureInfo ("en-US")) * 1024; + + if (speed_match.Groups [2].Value.Equals ("M")) + speed = speed * 1024; + + information = speed.ToSize (); + } + } return true; } @@ -416,6 +404,38 @@ namespace Sparkles.Git { } + StorageType? DetermineStorageType () + { + var git_ls_remote = new GitCommand (Configuration.DefaultConfiguration.TmpPath, + string.Format ("ls-remote --heads \"{0}\"", RemoteUrl), auth_info); + + string output = git_ls_remote.StartAndReadStandardOutput (); + + if (git_ls_remote.ExitCode != 0) + return null; + + if (string.IsNullOrWhiteSpace (output)) + return StorageType.Unknown; + + foreach (string line in output.Split ("\n".ToCharArray ())) { + string [] line_parts = line.Split ('/'); + string branch = line_parts [line_parts.Length - 1]; + + if (branch == "x-sparkleshare-lfs") + return StorageType.LargeFiles; + + string encrypted_storage_prefix = "x-sparkleshare-encrypted-"; + + if (branch.StartsWith (encrypted_storage_prefix)) { + password_salt = branch.Replace (encrypted_storage_prefix, ""); + return StorageType.Encrypted; + } + } + + return StorageType.Plain; + } + + void InstallExcludeRules () { string git_info_path = Path.Combine (TargetFolder, ".git", "info"); diff --git a/Sparkles/Git/GitRepository.cs b/Sparkles/Git/GitRepository.cs index a548a3b3..568c093c 100644 --- a/Sparkles/Git/GitRepository.cs +++ b/Sparkles/Git/GitRepository.cs @@ -229,6 +229,7 @@ namespace Sparkles.Git { // TODO: parse LFS progress while (!git_push.StandardError.EndOfStream) { string line = git_push.StandardError.ReadLine (); + Console.WriteLine (line); Match match = this.progress_regex.Match (line); double speed = 0.0; double number = 0.0;