beenull/SparkleShare
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

git fetcher: Implement LFS progress reporting

Browse source
This commit is contained in:
Hylke Bons 2016-06-28 09:41:17 +01:00
parent bc5698a6cc
commit 8ab758ccda
6 changed files with 201 additions and 123 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
SparkleShare/Common/BaseController.cs
View file

@ -88,7 +88,7 @@ namespace SparkleShare {
public delegate void FolderFetchErrorHandler (string remote_url, string [] errors);
public event FolderFetchingHandler FolderFetching = delegate { };
public delegate void FolderFetchingHandler (double percentage, double speed);
public delegate void FolderFetchingHandler (double percentage, double speed, string information);
public event Action FolderListChanged = delegate { };
@ -611,9 +611,9 @@ namespace SparkleShare {
}
void FetcherProgressChangedDelgate (double percentage, double speed)
void FetcherProgressChangedDelgate (double percentage, double speed, string information)
{
FolderFetching (percentage, speed);
FolderFetching (percentage, speed, information);
}

12
SparkleShare/Common/SetupController.cs
View file

@ -53,7 +53,7 @@ namespace SparkleShare {
public delegate void ChangePageEventHandler (PageType page, string [] warnings);
public event UpdateProgressBarEventHandler UpdateProgressBarEvent = delegate { };
public delegate void UpdateProgressBarEventHandler (double percentage, string speed);
public delegate void UpdateProgressBarEventHandler (double percentage, string information);
public event UpdateSetupContinueButtonEventHandler UpdateSetupContinueButtonEvent = delegate { };
public delegate void UpdateSetupContinueButtonEventHandler (bool button_enabled);
@ -370,14 +370,14 @@ namespace SparkleShare {
SparkleShare.Controller.FolderFetching -= SyncingPageFetchingDelegate;
}
private void SyncingPageFetchingDelegate (double percentage, double speed)
private void SyncingPageFetchingDelegate (double percentage, double speed ,string information)
{
ProgressBarPercentage = percentage;
if (speed == 0.0)
UpdateProgressBarEvent (ProgressBarPercentage, "");
else
UpdateProgressBarEvent (ProgressBarPercentage, "Fetching files… " + speed.ToSize () + "/s");
if (speed > 0)
information = speed.ToSize () + " – " + information;
UpdateProgressBarEvent (ProgressBarPercentage, information);
}

57
SparkleShare/Mac/UserInterface/Setup.cs
View file

@ -51,6 +51,12 @@ namespace SparkleShare {
private SparkleDataSource DataSource;
private NSButtonCell ButtonCellProto;
private NSMatrix Matrix;
public Setup () : base ()
{
Controller.HideWindowEvent += delegate {
@ -456,6 +462,57 @@ namespace SparkleShare {
Buttons.Add (CancelButton);
}
if (type == PageType.StorageSetup) {
Header = string.Format ("Storage type for ‘{0}’", Controller.SyncingFolder);
Description = "What type of storage would you like to use?";
ButtonCellProto = new NSButtonCell ();
ButtonCellProto.SetButtonType (NSButtonType.Radio);
ButtonCellProto.Font = NSFont.FromFontName (UserInterface.FontName + " Bold", NSFont.SystemFontSize);
Matrix = new NSMatrix (new RectangleF (215, 0, 256, 256), NSMatrixMode.Radio,
ButtonCellProto, SparkleShare.Controller.FetcherAvailableStorageTypes.Count, 1);
Matrix.BackgroundColor = NSColor.Yellow;
Matrix.CellSize = new SizeF (256, 18);
Matrix.IntercellSpacing = new SizeF (12, 12);
int i = 0;
foreach (StorageTypeInfo storage_type in SparkleShare.Controller.FetcherAvailableStorageTypes) {
Matrix.Cells [i].Title = storage_type.Name;
// Matrix.IntercellSpacing = new SizeF (30, 30);
// todo: description
i++;
}
ContentView.AddSubview (Matrix);
CancelButton = new NSButton () { Title = "Cancel" };
ContinueButton = new NSButton () { Title = "Continue" };
ContinueButton.Activated += delegate {
Console.WriteLine (Matrix.SelectedRow);
StorageTypeInfo selected_storage_type = SparkleShare.Controller.FetcherAvailableStorageTypes [Matrix.SelectedRow];
Controller.StoragePageCompleted (selected_storage_type.Type);
};
CancelButton.Activated += delegate { Controller.SyncingCancelled (); };
Buttons.Add (ContinueButton);
Buttons.Add (CancelButton);
MakeFirstResponder ((NSResponder)PasswordTextField);
NSApplication.SharedApplication.RequestUserAttention (NSRequestUserAttentionType.CriticalRequest);
}
if (type == PageType.CryptoSetup || type == PageType.CryptoPassword) {
if (type == PageType.CryptoSetup) {
Header = "Set up file encryption";

6
Sparkles/BaseFetcher.cs
View file

@ -45,7 +45,7 @@ namespace Sparkles {
public delegate void FinishedEventHandler (StorageType storage_type, string [] warnings);
public event ProgressChangedEventHandler ProgressChanged = delegate { };
public delegate void ProgressChangedEventHandler (double percentage, double speed);
public delegate void ProgressChangedEventHandler (double percentage, double speed, string information);
public abstract bool Fetch ();
@ -215,8 +215,8 @@ namespace Sparkles {
}
protected void OnProgressChanged (double percentage, double speed) {
ProgressChanged (percentage, speed);
protected void OnProgressChanged (double percentage, double speed, string information) {
ProgressChanged (percentage, speed, information);
}

242
Sparkles/Git/GitFetcher.cs
View file

@ -25,15 +25,11 @@ namespace Sparkles.Git {
public class GitFetcher : SSHFetcher {
SSHAuthenticationInfo auth_info;
GitCommand git_clone;
SSHAuthenticationInfo auth_info;
string password_salt = Path.GetRandomFileName ().SHA256 ().Substring (0, 16);
Regex progress_regex = new Regex (@"([0-9]+)%", RegexOptions.Compiled);
Regex speed_regex = new Regex (@"([0-9\.]+) ([KM])iB/s", RegexOptions.Compiled);
protected override bool IsFetchedRepoEmpty {
get {
@ -78,38 +74,6 @@ namespace Sparkles.Git {
}
StorageType? DetermineStorageType ()
{
var git_ls_remote = new GitCommand (Configuration.DefaultConfiguration.TmpPath,
string.Format ("ls-remote --heads \"{0}\"", RemoteUrl), auth_info);
string output = git_ls_remote.StartAndReadStandardOutput ();
if (git_ls_remote.ExitCode != 0)
return null;
if (string.IsNullOrWhiteSpace (output))
return StorageType.Unknown;
foreach (string line in output.Split ("\n".ToCharArray ())) {
string [] line_parts = line.Split ('/');
string branch = line_parts [line_parts.Length - 1];
if (branch == "x-sparkleshare-lfs")
return StorageType.LargeFiles;
string encrypted_storage_prefix = "x-sparkleshare-encrypted-";
if (branch.StartsWith (encrypted_storage_prefix)) {
password_salt = branch.Replace (encrypted_storage_prefix, "");
return StorageType.Encrypted;
}
}
return StorageType.Plain;
}
public override bool Fetch ()
{
if (!base.Fetch ())
@ -130,88 +94,48 @@ namespace Sparkles.Git {
if (storage_type == StorageType.LargeFiles)
git_clone_command = "lfs clone --progress --no-checkout";
var git_clone = new GitCommand (Configuration.DefaultConfiguration.TmpPath,
git_clone = new GitCommand (Configuration.DefaultConfiguration.TmpPath,
string.Format ("{0} \"{1}\" \"{2}\"", git_clone_command, RemoteUrl, TargetFolder),
auth_info);
git_clone.StartInfo.RedirectStandardError = true;
git_clone.Start ();
double percentage = 1.0;
StreamReader output_stream = git_clone.StandardError;
if (FetchedRepoStorageType == StorageType.LargeFiles)
output_stream = git_clone.StandardOutput;
var last_change = DateTime.Now;
var change_interval = new TimeSpan (0, 0, 0, 1);
var change_interval = new TimeSpan (0, 0, 0, 1);
try {
while (!git_clone.StandardError.EndOfStream) {
string line = git_clone.StandardError.ReadLine ();
Match match = progress_regex.Match (line);
double previous_percentage = 0;
double percentage = 0;
double speed = 0;
string information = "";
double number = 0.0;
double speed = 0.0;
if (match.Success) {
try {
number = double.Parse (match.Groups [1].Value, new CultureInfo ("en-US"));
while (!output_stream.EndOfStream) {
string line = output_stream.ReadLine ();
} catch (FormatException) {
Logger.LogInfo ("Git", "Error parsing progress: \"" + match.Groups [1] + "\"");
}
previous_percentage = percentage;
bool parse_success = ParseProgress (line, out percentage, out speed, out information);
// The pushing progress consists of two stages: the "Compressing
// objects" stage which we count as 20% of the total progress, and
// the "Writing objects" stage which we count as the last 80%
if (line.Contains ("Compressing")) {
// "Compressing objects" stage
number = (number / 100 * 20);
if (!parse_success) {
IsActive = false;
git_clone.Kill ();
git_clone.Dispose ();
} else {
// "Writing objects" stage
number = (number / 100 * 80 + 20);
Match speed_match = speed_regex.Match (line);
if (speed_match.Success) {
try {
speed = double.Parse (speed_match.Groups [1].Value, new CultureInfo ("en-US")) * 1024;
} catch (FormatException) {
Logger.LogInfo ("Git", "Error parsing speed: \"" + speed_match.Groups [1] + "\"");
}
if (speed_match.Groups [2].Value.Equals ("M"))
speed = speed * 1024;
}
}
} else {
Logger.LogInfo ("Fetcher", line);
line = line.Trim (new char [] {' ', '@'});
if (line.StartsWith ("fatal:", StringComparison.InvariantCultureIgnoreCase) ||
line.StartsWith ("error:", StringComparison.InvariantCultureIgnoreCase)) {
errors.Add (line);
} else if (line.StartsWith ("WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!")) {
errors.Add ("warning: Remote host identification has changed!");
} else if (line.StartsWith ("WARNING: POSSIBLE DNS SPOOFING DETECTED!")) {
errors.Add ("warning: Possible DNS spoofing detected!");
}
}
if (number >= percentage) {
percentage = number;
if (DateTime.Compare (last_change, DateTime.Now.Subtract (change_interval)) < 0) {
OnProgressChanged (percentage, speed);
last_change = DateTime.Now;
}
}
return false;
}
} catch (Exception) {
IsActive = false;
return false;
if (percentage <= previous_percentage)
continue;
if (DateTime.Compare (last_change, DateTime.Now.Subtract (change_interval)) < 0) {
Console.WriteLine (percentage);
OnProgressChanged (percentage, speed, information);
last_change = DateTime.Now;
}
}
git_clone.WaitForExit ();
@ -219,17 +143,81 @@ namespace Sparkles.Git {
if (git_clone.ExitCode != 0)
return false;
while (percentage < 100) {
percentage += 25;
Thread.Sleep (500);
OnProgressChanged (100, 0, "");
Thread.Sleep (500);
if (percentage >= 100)
break;
return true;
}
Thread.Sleep (500);
OnProgressChanged (percentage, 0);
Regex progress_regex = new Regex (@"([0-9]+)%", RegexOptions.Compiled);
Regex progress_regex_lfs = new Regex (@"^Git LFS:.*([0-9]+) of ([0-9]+).*", RegexOptions.Compiled);
Regex speed_regex = new Regex (@"([0-9\.]+) ([KM])iB/s", RegexOptions.Compiled);
public bool ParseProgress (string line, out double percentage, out double speed, out string information)
{
percentage = 0;
speed = 0;
information = "";
Match match;
if (FetchedRepoStorageType == StorageType.LargeFiles) {
match = progress_regex_lfs.Match (line);
if (!match.Success)
return true;
percentage = double.Parse (match.Groups [1].Value) / double.Parse (match.Groups [2].Value) * 100;
information = string.Format ("{0} of {1} files", match.Groups [1].Value, match.Groups [2].Value);
return true;
}
OnProgressChanged (100, 0);
match = progress_regex.Match (line);
if (!match.Success) {
Logger.LogInfo ("Fetcher", line);
line = line.Trim (new char [] { ' ', '@' });
if (line.StartsWith ("fatal:", StringComparison.InvariantCultureIgnoreCase) ||
line.StartsWith ("error:", StringComparison.InvariantCultureIgnoreCase)) {
errors.Add (line);
} else if (line.StartsWith ("WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!")) {
errors.Add ("warning: Remote host identification has changed");
} else if (line.StartsWith ("WARNING: POSSIBLE DNS SPOOFING DETECTED!")) {
errors.Add ("warning: Possible DNS spoofing detected");
}
return false;
}
int number = int.Parse (match.Groups [1].Value, new CultureInfo ("en-US"));
// The pushing progress consists of two stages: the "Compressing
// objects" stage which we count as 20% of the total progress, and
// the "Writing objects" stage which we count as the last 80%
if (line.Contains ("Compressing objects")) {
// "Compressing objects" stage
percentage = (number / 100 * 20);
} else if (line.Contains ("Writing objects")) {
percentage = (number / 100 * 80 + 20);
Match speed_match = speed_regex.Match (line);
if (speed_match.Success) {
speed = double.Parse (speed_match.Groups [1].Value, new CultureInfo ("en-US")) * 1024;
if (speed_match.Groups [2].Value.Equals ("M"))
speed = speed * 1024;
information = speed.ToSize ();
}
}
return true;
}
@ -416,6 +404,38 @@ namespace Sparkles.Git {
}
StorageType? DetermineStorageType ()
{
var git_ls_remote = new GitCommand (Configuration.DefaultConfiguration.TmpPath,
string.Format ("ls-remote --heads \"{0}\"", RemoteUrl), auth_info);
string output = git_ls_remote.StartAndReadStandardOutput ();
if (git_ls_remote.ExitCode != 0)
return null;
if (string.IsNullOrWhiteSpace (output))
return StorageType.Unknown;
foreach (string line in output.Split ("\n".ToCharArray ())) {
string [] line_parts = line.Split ('/');
string branch = line_parts [line_parts.Length - 1];
if (branch == "x-sparkleshare-lfs")
return StorageType.LargeFiles;
string encrypted_storage_prefix = "x-sparkleshare-encrypted-";
if (branch.StartsWith (encrypted_storage_prefix)) {
password_salt = branch.Replace (encrypted_storage_prefix, "");
return StorageType.Encrypted;
}
}
return StorageType.Plain;
}
void InstallExcludeRules ()
{
string git_info_path = Path.Combine (TargetFolder, ".git", "info");

1
Sparkles/Git/GitRepository.cs
View file

@ -229,6 +229,7 @@ namespace Sparkles.Git {
// TODO: parse LFS progress
while (!git_push.StandardError.EndOfStream) {
string line = git_push.StandardError.ReadLine ();
Console.WriteLine (line);
Match match = this.progress_regex.Match (line);
double speed = 0.0;
double number = 0.0;