git fetcher: Implement LFS progress reporting
This commit is contained in:
parent
bc5698a6cc
commit
8ab758ccda
|
@ -88,7 +88,7 @@ namespace SparkleShare {
|
|||
public delegate void FolderFetchErrorHandler (string remote_url, string [] errors);
|
||||
|
||||
public event FolderFetchingHandler FolderFetching = delegate { };
|
||||
public delegate void FolderFetchingHandler (double percentage, double speed);
|
||||
public delegate void FolderFetchingHandler (double percentage, double speed, string information);
|
||||
|
||||
|
||||
public event Action FolderListChanged = delegate { };
|
||||
|
@ -611,9 +611,9 @@ namespace SparkleShare {
|
|||
}
|
||||
|
||||
|
||||
void FetcherProgressChangedDelgate (double percentage, double speed)
|
||||
void FetcherProgressChangedDelgate (double percentage, double speed, string information)
|
||||
{
|
||||
FolderFetching (percentage, speed);
|
||||
FolderFetching (percentage, speed, information);
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -53,7 +53,7 @@ namespace SparkleShare {
|
|||
public delegate void ChangePageEventHandler (PageType page, string [] warnings);
|
||||
|
||||
public event UpdateProgressBarEventHandler UpdateProgressBarEvent = delegate { };
|
||||
public delegate void UpdateProgressBarEventHandler (double percentage, string speed);
|
||||
public delegate void UpdateProgressBarEventHandler (double percentage, string information);
|
||||
|
||||
public event UpdateSetupContinueButtonEventHandler UpdateSetupContinueButtonEvent = delegate { };
|
||||
public delegate void UpdateSetupContinueButtonEventHandler (bool button_enabled);
|
||||
|
@ -370,14 +370,14 @@ namespace SparkleShare {
|
|||
SparkleShare.Controller.FolderFetching -= SyncingPageFetchingDelegate;
|
||||
}
|
||||
|
||||
private void SyncingPageFetchingDelegate (double percentage, double speed)
|
||||
private void SyncingPageFetchingDelegate (double percentage, double speed ,string information)
|
||||
{
|
||||
ProgressBarPercentage = percentage;
|
||||
|
||||
if (speed == 0.0)
|
||||
UpdateProgressBarEvent (ProgressBarPercentage, "");
|
||||
else
|
||||
UpdateProgressBarEvent (ProgressBarPercentage, "Fetching files… " + speed.ToSize () + "/s");
|
||||
if (speed > 0)
|
||||
information = speed.ToSize () + " – " + information;
|
||||
|
||||
UpdateProgressBarEvent (ProgressBarPercentage, information);
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -51,6 +51,12 @@ namespace SparkleShare {
|
|||
private SparkleDataSource DataSource;
|
||||
|
||||
|
||||
|
||||
private NSButtonCell ButtonCellProto;
|
||||
|
||||
private NSMatrix Matrix;
|
||||
|
||||
|
||||
public Setup () : base ()
|
||||
{
|
||||
Controller.HideWindowEvent += delegate {
|
||||
|
@ -456,6 +462,57 @@ namespace SparkleShare {
|
|||
Buttons.Add (CancelButton);
|
||||
}
|
||||
|
||||
|
||||
|
||||
if (type == PageType.StorageSetup) {
|
||||
Header = string.Format ("Storage type for ‘{0}’", Controller.SyncingFolder);
|
||||
Description = "What type of storage would you like to use?";
|
||||
|
||||
|
||||
ButtonCellProto = new NSButtonCell ();
|
||||
ButtonCellProto.SetButtonType (NSButtonType.Radio);
|
||||
ButtonCellProto.Font = NSFont.FromFontName (UserInterface.FontName + " Bold", NSFont.SystemFontSize);
|
||||
|
||||
|
||||
Matrix = new NSMatrix (new RectangleF (215, 0, 256, 256), NSMatrixMode.Radio,
|
||||
ButtonCellProto, SparkleShare.Controller.FetcherAvailableStorageTypes.Count, 1);
|
||||
Matrix.BackgroundColor = NSColor.Yellow;
|
||||
Matrix.CellSize = new SizeF (256, 18);
|
||||
Matrix.IntercellSpacing = new SizeF (12, 12);
|
||||
|
||||
int i = 0;
|
||||
foreach (StorageTypeInfo storage_type in SparkleShare.Controller.FetcherAvailableStorageTypes) {
|
||||
Matrix.Cells [i].Title = storage_type.Name;
|
||||
|
||||
// Matrix.IntercellSpacing = new SizeF (30, 30);
|
||||
// todo: description
|
||||
i++;
|
||||
}
|
||||
|
||||
ContentView.AddSubview (Matrix);
|
||||
|
||||
|
||||
CancelButton = new NSButton () { Title = "Cancel" };
|
||||
ContinueButton = new NSButton () { Title = "Continue" };
|
||||
|
||||
|
||||
ContinueButton.Activated += delegate {
|
||||
Console.WriteLine (Matrix.SelectedRow);
|
||||
|
||||
StorageTypeInfo selected_storage_type = SparkleShare.Controller.FetcherAvailableStorageTypes [Matrix.SelectedRow];
|
||||
Controller.StoragePageCompleted (selected_storage_type.Type);
|
||||
};
|
||||
|
||||
CancelButton.Activated += delegate { Controller.SyncingCancelled (); };
|
||||
|
||||
|
||||
Buttons.Add (ContinueButton);
|
||||
Buttons.Add (CancelButton);
|
||||
|
||||
MakeFirstResponder ((NSResponder)PasswordTextField);
|
||||
NSApplication.SharedApplication.RequestUserAttention (NSRequestUserAttentionType.CriticalRequest);
|
||||
}
|
||||
|
||||
if (type == PageType.CryptoSetup || type == PageType.CryptoPassword) {
|
||||
if (type == PageType.CryptoSetup) {
|
||||
Header = "Set up file encryption";
|
||||
|
|
|
@ -45,7 +45,7 @@ namespace Sparkles {
|
|||
public delegate void FinishedEventHandler (StorageType storage_type, string [] warnings);
|
||||
|
||||
public event ProgressChangedEventHandler ProgressChanged = delegate { };
|
||||
public delegate void ProgressChangedEventHandler (double percentage, double speed);
|
||||
public delegate void ProgressChangedEventHandler (double percentage, double speed, string information);
|
||||
|
||||
|
||||
public abstract bool Fetch ();
|
||||
|
@ -215,8 +215,8 @@ namespace Sparkles {
|
|||
}
|
||||
|
||||
|
||||
protected void OnProgressChanged (double percentage, double speed) {
|
||||
ProgressChanged (percentage, speed);
|
||||
protected void OnProgressChanged (double percentage, double speed, string information) {
|
||||
ProgressChanged (percentage, speed, information);
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -25,15 +25,11 @@ namespace Sparkles.Git {
|
|||
|
||||
public class GitFetcher : SSHFetcher {
|
||||
|
||||
SSHAuthenticationInfo auth_info;
|
||||
GitCommand git_clone;
|
||||
SSHAuthenticationInfo auth_info;
|
||||
|
||||
string password_salt = Path.GetRandomFileName ().SHA256 ().Substring (0, 16);
|
||||
|
||||
Regex progress_regex = new Regex (@"([0-9]+)%", RegexOptions.Compiled);
|
||||
Regex speed_regex = new Regex (@"([0-9\.]+) ([KM])iB/s", RegexOptions.Compiled);
|
||||
|
||||
|
||||
|
||||
protected override bool IsFetchedRepoEmpty {
|
||||
get {
|
||||
|
@ -78,38 +74,6 @@ namespace Sparkles.Git {
|
|||
}
|
||||
|
||||
|
||||
StorageType? DetermineStorageType ()
|
||||
{
|
||||
var git_ls_remote = new GitCommand (Configuration.DefaultConfiguration.TmpPath,
|
||||
string.Format ("ls-remote --heads \"{0}\"", RemoteUrl), auth_info);
|
||||
|
||||
string output = git_ls_remote.StartAndReadStandardOutput ();
|
||||
|
||||
if (git_ls_remote.ExitCode != 0)
|
||||
return null;
|
||||
|
||||
if (string.IsNullOrWhiteSpace (output))
|
||||
return StorageType.Unknown;
|
||||
|
||||
foreach (string line in output.Split ("\n".ToCharArray ())) {
|
||||
string [] line_parts = line.Split ('/');
|
||||
string branch = line_parts [line_parts.Length - 1];
|
||||
|
||||
if (branch == "x-sparkleshare-lfs")
|
||||
return StorageType.LargeFiles;
|
||||
|
||||
string encrypted_storage_prefix = "x-sparkleshare-encrypted-";
|
||||
|
||||
if (branch.StartsWith (encrypted_storage_prefix)) {
|
||||
password_salt = branch.Replace (encrypted_storage_prefix, "");
|
||||
return StorageType.Encrypted;
|
||||
}
|
||||
}
|
||||
|
||||
return StorageType.Plain;
|
||||
}
|
||||
|
||||
|
||||
public override bool Fetch ()
|
||||
{
|
||||
if (!base.Fetch ())
|
||||
|
@ -130,59 +94,90 @@ namespace Sparkles.Git {
|
|||
if (storage_type == StorageType.LargeFiles)
|
||||
git_clone_command = "lfs clone --progress --no-checkout";
|
||||
|
||||
var git_clone = new GitCommand (Configuration.DefaultConfiguration.TmpPath,
|
||||
git_clone = new GitCommand (Configuration.DefaultConfiguration.TmpPath,
|
||||
string.Format ("{0} \"{1}\" \"{2}\"", git_clone_command, RemoteUrl, TargetFolder),
|
||||
auth_info);
|
||||
|
||||
git_clone.StartInfo.RedirectStandardError = true;
|
||||
git_clone.Start ();
|
||||
|
||||
double percentage = 1.0;
|
||||
StreamReader output_stream = git_clone.StandardError;
|
||||
|
||||
if (FetchedRepoStorageType == StorageType.LargeFiles)
|
||||
output_stream = git_clone.StandardOutput;
|
||||
|
||||
var last_change = DateTime.Now;
|
||||
var change_interval = new TimeSpan (0, 0, 0, 1);
|
||||
|
||||
try {
|
||||
while (!git_clone.StandardError.EndOfStream) {
|
||||
string line = git_clone.StandardError.ReadLine ();
|
||||
Match match = progress_regex.Match (line);
|
||||
double previous_percentage = 0;
|
||||
double percentage = 0;
|
||||
double speed = 0;
|
||||
string information = "";
|
||||
|
||||
double number = 0.0;
|
||||
double speed = 0.0;
|
||||
if (match.Success) {
|
||||
try {
|
||||
number = double.Parse (match.Groups [1].Value, new CultureInfo ("en-US"));
|
||||
while (!output_stream.EndOfStream) {
|
||||
string line = output_stream.ReadLine ();
|
||||
|
||||
} catch (FormatException) {
|
||||
Logger.LogInfo ("Git", "Error parsing progress: \"" + match.Groups [1] + "\"");
|
||||
previous_percentage = percentage;
|
||||
bool parse_success = ParseProgress (line, out percentage, out speed, out information);
|
||||
|
||||
if (!parse_success) {
|
||||
IsActive = false;
|
||||
git_clone.Kill ();
|
||||
git_clone.Dispose ();
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// The pushing progress consists of two stages: the "Compressing
|
||||
// objects" stage which we count as 20% of the total progress, and
|
||||
// the "Writing objects" stage which we count as the last 80%
|
||||
if (line.Contains ("Compressing")) {
|
||||
// "Compressing objects" stage
|
||||
number = (number / 100 * 20);
|
||||
if (percentage <= previous_percentage)
|
||||
continue;
|
||||
|
||||
} else {
|
||||
// "Writing objects" stage
|
||||
number = (number / 100 * 80 + 20);
|
||||
Match speed_match = speed_regex.Match (line);
|
||||
|
||||
if (speed_match.Success) {
|
||||
try {
|
||||
speed = double.Parse (speed_match.Groups [1].Value, new CultureInfo ("en-US")) * 1024;
|
||||
|
||||
} catch (FormatException) {
|
||||
Logger.LogInfo ("Git", "Error parsing speed: \"" + speed_match.Groups [1] + "\"");
|
||||
}
|
||||
|
||||
if (speed_match.Groups [2].Value.Equals ("M"))
|
||||
speed = speed * 1024;
|
||||
if (DateTime.Compare (last_change, DateTime.Now.Subtract (change_interval)) < 0) {
|
||||
Console.WriteLine (percentage);
|
||||
OnProgressChanged (percentage, speed, information);
|
||||
last_change = DateTime.Now;
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
git_clone.WaitForExit ();
|
||||
|
||||
if (git_clone.ExitCode != 0)
|
||||
return false;
|
||||
|
||||
Thread.Sleep (500);
|
||||
OnProgressChanged (100, 0, "");
|
||||
Thread.Sleep (500);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
Regex progress_regex = new Regex (@"([0-9]+)%", RegexOptions.Compiled);
|
||||
Regex progress_regex_lfs = new Regex (@"^Git LFS:.*([0-9]+) of ([0-9]+).*", RegexOptions.Compiled);
|
||||
Regex speed_regex = new Regex (@"([0-9\.]+) ([KM])iB/s", RegexOptions.Compiled);
|
||||
|
||||
public bool ParseProgress (string line, out double percentage, out double speed, out string information)
|
||||
{
|
||||
percentage = 0;
|
||||
speed = 0;
|
||||
information = "";
|
||||
|
||||
Match match;
|
||||
|
||||
if (FetchedRepoStorageType == StorageType.LargeFiles) {
|
||||
match = progress_regex_lfs.Match (line);
|
||||
|
||||
if (!match.Success)
|
||||
return true;
|
||||
|
||||
percentage = double.Parse (match.Groups [1].Value) / double.Parse (match.Groups [2].Value) * 100;
|
||||
information = string.Format ("{0} of {1} files", match.Groups [1].Value, match.Groups [2].Value);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
match = progress_regex.Match (line);
|
||||
|
||||
if (!match.Success) {
|
||||
Logger.LogInfo ("Fetcher", line);
|
||||
line = line.Trim (new char [] { ' ', '@' });
|
||||
|
||||
|
@ -192,44 +187,37 @@ namespace Sparkles.Git {
|
|||
errors.Add (line);
|
||||
|
||||
} else if (line.StartsWith ("WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!")) {
|
||||
errors.Add ("warning: Remote host identification has changed!");
|
||||
errors.Add ("warning: Remote host identification has changed");
|
||||
|
||||
} else if (line.StartsWith ("WARNING: POSSIBLE DNS SPOOFING DETECTED!")) {
|
||||
errors.Add ("warning: Possible DNS spoofing detected!");
|
||||
}
|
||||
errors.Add ("warning: Possible DNS spoofing detected");
|
||||
}
|
||||
|
||||
if (number >= percentage) {
|
||||
percentage = number;
|
||||
|
||||
if (DateTime.Compare (last_change, DateTime.Now.Subtract (change_interval)) < 0) {
|
||||
OnProgressChanged (percentage, speed);
|
||||
last_change = DateTime.Now;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} catch (Exception) {
|
||||
IsActive = false;
|
||||
return false;
|
||||
}
|
||||
|
||||
git_clone.WaitForExit ();
|
||||
int number = int.Parse (match.Groups [1].Value, new CultureInfo ("en-US"));
|
||||
|
||||
if (git_clone.ExitCode != 0)
|
||||
return false;
|
||||
// The pushing progress consists of two stages: the "Compressing
|
||||
// objects" stage which we count as 20% of the total progress, and
|
||||
// the "Writing objects" stage which we count as the last 80%
|
||||
if (line.Contains ("Compressing objects")) {
|
||||
// "Compressing objects" stage
|
||||
percentage = (number / 100 * 20);
|
||||
|
||||
while (percentage < 100) {
|
||||
percentage += 25;
|
||||
} else if (line.Contains ("Writing objects")) {
|
||||
percentage = (number / 100 * 80 + 20);
|
||||
Match speed_match = speed_regex.Match (line);
|
||||
|
||||
if (percentage >= 100)
|
||||
break;
|
||||
if (speed_match.Success) {
|
||||
speed = double.Parse (speed_match.Groups [1].Value, new CultureInfo ("en-US")) * 1024;
|
||||
|
||||
Thread.Sleep (500);
|
||||
OnProgressChanged (percentage, 0);
|
||||
if (speed_match.Groups [2].Value.Equals ("M"))
|
||||
speed = speed * 1024;
|
||||
|
||||
information = speed.ToSize ();
|
||||
}
|
||||
}
|
||||
|
||||
OnProgressChanged (100, 0);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
@ -416,6 +404,38 @@ namespace Sparkles.Git {
|
|||
}
|
||||
|
||||
|
||||
StorageType? DetermineStorageType ()
|
||||
{
|
||||
var git_ls_remote = new GitCommand (Configuration.DefaultConfiguration.TmpPath,
|
||||
string.Format ("ls-remote --heads \"{0}\"", RemoteUrl), auth_info);
|
||||
|
||||
string output = git_ls_remote.StartAndReadStandardOutput ();
|
||||
|
||||
if (git_ls_remote.ExitCode != 0)
|
||||
return null;
|
||||
|
||||
if (string.IsNullOrWhiteSpace (output))
|
||||
return StorageType.Unknown;
|
||||
|
||||
foreach (string line in output.Split ("\n".ToCharArray ())) {
|
||||
string [] line_parts = line.Split ('/');
|
||||
string branch = line_parts [line_parts.Length - 1];
|
||||
|
||||
if (branch == "x-sparkleshare-lfs")
|
||||
return StorageType.LargeFiles;
|
||||
|
||||
string encrypted_storage_prefix = "x-sparkleshare-encrypted-";
|
||||
|
||||
if (branch.StartsWith (encrypted_storage_prefix)) {
|
||||
password_salt = branch.Replace (encrypted_storage_prefix, "");
|
||||
return StorageType.Encrypted;
|
||||
}
|
||||
}
|
||||
|
||||
return StorageType.Plain;
|
||||
}
|
||||
|
||||
|
||||
void InstallExcludeRules ()
|
||||
{
|
||||
string git_info_path = Path.Combine (TargetFolder, ".git", "info");
|
||||
|
|
|
@ -229,6 +229,7 @@ namespace Sparkles.Git {
|
|||
// TODO: parse LFS progress
|
||||
while (!git_push.StandardError.EndOfStream) {
|
||||
string line = git_push.StandardError.ReadLine ();
|
||||
Console.WriteLine (line);
|
||||
Match match = this.progress_regex.Match (line);
|
||||
double speed = 0.0;
|
||||
double number = 0.0;
|
||||
|
|
Loading…
Reference in a new issue