beenull/SparkleShare
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

git: Simplify progress parsing and support LFS

Browse source
This commit is contained in:
Hylke Bons 2016-06-29 14:44:15 +01:00
parent e3e99d20b9
commit a6c57e51b7
8 changed files with 203 additions and 296 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
SparkleShare/Common/BaseController.cs
View file

@ -67,11 +67,12 @@ namespace SparkleShare {
public bool RepositoriesLoaded { get; private set; }
public string FoldersPath { get; private set; }
public double ProgressPercentage = 0.0;
public double ProgressSpeedUp = 0.0;
public double ProgressSpeedDown = 0.0;
public double ProgressPercentage = 0.0;
public double ProgressSpeedUp = 0.0;
public double ProgressSpeedDown = 0.0;
public string ProgressInformation = "";
public event ShowSetupWindowEventHandler ShowSetupWindowEvent = delegate { };
public delegate void ShowSetupWindowEventHandler (PageType page_type);
@ -417,18 +418,20 @@ namespace SparkleShare {
repo.SyncStatusChanged += delegate (SyncStatus status) {
if (status == SyncStatus.Idle) {
ProgressPercentage = 0.0;
ProgressSpeedUp = 0.0;
ProgressSpeedDown = 0.0;
ProgressPercentage = 0.0;
ProgressSpeedUp = 0.0;
ProgressSpeedDown = 0.0;
ProgressInformation = "";
}
UpdateState ();
};
repo.ProgressChanged += delegate {
ProgressPercentage = 0.0;
ProgressSpeedUp = 0.0;
ProgressSpeedDown = 0.0;
ProgressPercentage = 0.0;
ProgressSpeedUp = 0.0;
ProgressSpeedDown = 0.0;
ProgressInformation = "";
double percentage = 0.0;
int repo_count = 0;
@ -445,6 +448,9 @@ namespace SparkleShare {
if (rep.Status == SyncStatus.SyncDown)
ProgressSpeedDown += rep.ProgressSpeed;
}
if (repo_count == 1)
ProgressInformation = repo.ProgressInformation;
if (repo_count > 0)
ProgressPercentage = percentage / repo_count;
@ -460,8 +466,7 @@ namespace SparkleShare {
};
repo.ConflictResolved += delegate {
AlertNotificationRaised ("Resolved a file collision",
"Local and server versions were kept.");
AlertNotificationRaised ("Resolved a file collision", "Local and server versions were kept.");
};
AddRepository (repo);

48
SparkleShare/Common/StatusIconController.cs
View file

@ -133,38 +133,11 @@ namespace SparkleShare {
public delegate void UpdateQuitItemEventHandler (bool quit_item_enabled);
public IconState CurrentState = IconState.Idle;
public string StateText = "Welcome to SparkleShare!";
public string StateText = "Welcome to SparkleShare!";
public ProjectInfo [] Projects = new ProjectInfo [0];
public int ProgressPercentage {
get {
return (int) SparkleShare.Controller.ProgressPercentage;
}
}
public string ProgressSpeed {
get {
string progress_speed = "";
if (SparkleShare.Controller.ProgressSpeedDown == 0.0 && SparkleShare.Controller.ProgressSpeedUp > 0.0) {
progress_speed = SparkleShare.Controller.ProgressSpeedUp.ToSize () + "/s ";
} else if (SparkleShare.Controller.ProgressSpeedUp == 0.0 && SparkleShare.Controller.ProgressSpeedDown > 0.0) {
progress_speed = SparkleShare.Controller.ProgressSpeedDown.ToSize () + "/s ";
} else if (SparkleShare.Controller.ProgressSpeedUp > 0.0 &&
SparkleShare.Controller.ProgressSpeedDown > 0.0) {
progress_speed = "Up: " + SparkleShare.Controller.ProgressSpeedUp.ToSize () + "/s " +
"Down: " + SparkleShare.Controller.ProgressSpeedDown.ToSize () + "/s";
}
return progress_speed;
}
}
public bool RecentEventsItemEnabled {
get {
return (SparkleShare.Controller.Repositories.Length > 0);
@ -242,8 +215,23 @@ namespace SparkleShare {
StateText = "Receiving…";
}
if (ProgressPercentage > 0)
StateText += " " + ProgressPercentage + "% " + ProgressSpeed;
int progress_percentage = (int) SparkleShare.Controller.ProgressPercentage;
string progress_speed = "";
if (SparkleShare.Controller.ProgressSpeedUp > 0.0 && SparkleShare.Controller.ProgressSpeedDown > 0.0) {
progress_speed = "Up: " + SparkleShare.Controller.ProgressSpeedUp.ToSize () + "/s " +
"Down: " + SparkleShare.Controller.ProgressSpeedDown.ToSize () + "/s";
}
if (SparkleShare.Controller.ProgressSpeedUp > 0.0)
progress_speed = SparkleShare.Controller.ProgressSpeedUp.ToSize () + "/s ";
if (SparkleShare.Controller.ProgressSpeedDown > 0.0)
progress_speed = SparkleShare.Controller.ProgressSpeedDown.ToSize () + "/s ";
if (progress_percentage > 0)
StateText += string.Format (" {0}% {1} – {2}",
progress_percentage, progress_speed, SparkleShare.Controller.ProgressInformation);
UpdateIconEvent (CurrentState);
UpdateStatusItemEvent (StateText);

6
Sparkles/BaseFetcher.cs
View file

@ -215,7 +215,13 @@ namespace Sparkles {
}
DateTime progress_last_change = DateTime.Now;
protected void OnProgressChanged (double percentage, double speed, string information) {
// Only trigger the ProgressChanged event once per second
if (DateTime.Compare (this.progress_last_change, DateTime.Now.Subtract (new TimeSpan (0, 0, 0, 1))) >= 0)
return;
ProgressChanged (percentage, speed, information);
}

28
Sparkles/BaseRepository.cs
View file

@ -65,7 +65,8 @@ namespace Sparkles {
DiskSpaceExceeded,
UnreadableFiles,
NotFound,
IncompatibleClientServer
IncompatibleClientServer,
Unknown
}
@ -112,8 +113,10 @@ namespace Sparkles {
public SyncStatus Status { get; set; }
public ErrorStatus Error { get; protected set; }
public bool IsBuffering { get; set; }
public double ProgressPercentage { get; set; }
public double ProgressSpeed { get; set; }
public double ProgressPercentage { get; private set; }
public double ProgressSpeed { get; private set; }
public string ProgressInformation { get; private set; }
public DateTime LastSync {
get {
@ -163,7 +166,6 @@ namespace Sparkles {
Watcher watcher;
TimeSpan poll_interval = PollInterval.Short;
DateTime last_poll = DateTime.Now;
DateTime progress_last_change = DateTime.Now;
Timers.Timer remote_timer = new Timers.Timer () { Interval = 5000 };
DisconnectReason last_disconnect_reason = DisconnectReason.None;
@ -365,21 +367,25 @@ namespace Sparkles {
}
protected void OnProgressChanged (double progress_percentage, double progress_speed)
DateTime progress_last_change = DateTime.Now;
protected void OnProgressChanged (double percentage, double speed, string information)
{
if (progress_percentage < 1)
if (percentage < 1)
return;
// Only trigger the ProgressChanged event once per second
if (DateTime.Compare (this.progress_last_change, DateTime.Now.Subtract (new TimeSpan (0, 0, 0, 1))) >= 0)
return;
if (progress_percentage == 100.0)
progress_percentage = 99.0;
if (percentage == 100.0)
percentage = 99.0;
ProgressPercentage = progress_percentage;
ProgressSpeed = progress_speed;
this.progress_last_change = DateTime.Now;
progress_last_change = DateTime.Now;
ProgressPercentage = percentage;
ProgressSpeed = speed;
ProgressInformation = information;
ProgressChanged ();
}

1
Sparkles/Command.cs
View file

@ -18,7 +18,6 @@
using System;
using System.Diagnostics;
using System.IO;
using System.Reflection;
namespace Sparkles {

95
Sparkles/Git/GitCommand.cs
View file

@ -14,7 +14,8 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
using System;
using System.Globalization;
using System.Text.RegularExpressions;
namespace Sparkles.Git {
@ -95,6 +96,98 @@ namespace Sparkles.Git {
}
static Regex progress_regex = new Regex (@"([0-9]+)%", RegexOptions.Compiled);
static Regex progress_regex_lfs = new Regex (@".*([0-9]+) of ([0-9]+).*", RegexOptions.Compiled);
static Regex speed_regex = new Regex (@"([0-9\.]+) ([KM])iB/s", RegexOptions.Compiled);
public static ErrorStatus ParseProgress (string line, out double percentage, out double speed, out string information)
{
percentage = 0;
speed = 0;
information = "";
Match match;
if (line.StartsWith ("Git LFS:")) {
match = progress_regex_lfs.Match (line);
if (!match.Success)
return ErrorStatus.None;
percentage = double.Parse (match.Groups [1].Value) / double.Parse (match.Groups [2].Value) * 100;
information = string.Format ("{0} of {1} files", match.Groups [1].Value, match.Groups [2].Value);
return ErrorStatus.None;
}
match = progress_regex.Match (line);
if (!match.Success && !string.IsNullOrWhiteSpace (line)) {
Logger.LogInfo ("Git", line);
return FindError (line);
}
int number = int.Parse (match.Groups [1].Value);
// The pushing progress consists of two stages: the "Compressing
// objects" stage which we count as 20% of the total progress, and
// the "Writing objects" stage which we count as the last 80%
if (line.Contains ("Compressing objects")) {
// "Compressing objects" stage
percentage = (number / 100 * 20);
} else if (line.Contains ("Writing objects")) {
percentage = (number / 100 * 80 + 20);
Match speed_match = speed_regex.Match (line);
if (speed_match.Success) {
speed = double.Parse (speed_match.Groups [1].Value, new CultureInfo ("en-US")) * 1024;
if (speed_match.Groups [2].Value.Equals ("M"))
speed = speed * 1024;
information = speed.ToSize ();
}
}
return ErrorStatus.None;
}
static ErrorStatus FindError (string line)
{
ErrorStatus error = ErrorStatus.None;
if (line.Contains ("WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!") ||
line.Contains ("WARNING: POSSIBLE DNS SPOOFING DETECTED!")) {
error = ErrorStatus.HostIdentityChanged;
}
if (line.StartsWith ("Permission denied") ||
line.StartsWith ("ssh_exchange_identification: Connection closed by remote host") ||
line.StartsWith ("The authenticity of host")) {
error = ErrorStatus.AuthenticationFailed;
}
if (line.EndsWith ("does not appear to be a git repository"))
error = ErrorStatus.NotFound;
if (line.EndsWith ("expected old/new/ref, got 'shallow"))
error = ErrorStatus.IncompatibleClientServer;
if (line.StartsWith ("error: Disk space exceeded") ||
line.EndsWith ("No space left on device") ||
line.EndsWith ("file write error (Disk quota exceeded)")) {
error = ErrorStatus.DiskSpaceExceeded;
}
return error;
}
public static string FormatGitSSHCommand (SSHAuthenticationInfo auth_info)
{
return SSHPath + " " +

85
Sparkles/Git/GitFetcher.cs
View file

@ -106,9 +106,6 @@ namespace Sparkles.Git {
if (FetchedRepoStorageType == StorageType.LargeFiles)
output_stream = git_clone.StandardOutput;
var last_change = DateTime.Now;
var change_interval = new TimeSpan (0, 0, 0, 1);
double previous_percentage = 0;
double percentage = 0;
double speed = 0;
@ -118,9 +115,9 @@ namespace Sparkles.Git {
string line = output_stream.ReadLine ();
previous_percentage = percentage;
bool parse_success = ParseProgress (line, out percentage, out speed, out information);
ErrorStatus error = GitCommand.ParseProgress (line, out percentage, out speed, out information);
if (!parse_success) {
if (error != ErrorStatus.None) {
IsActive = false;
git_clone.Kill ();
git_clone.Dispose ();
@ -130,12 +127,6 @@ namespace Sparkles.Git {
if (percentage <= previous_percentage)
continue;
if (DateTime.Compare (last_change, DateTime.Now.Subtract (change_interval)) < 0) {
Console.WriteLine (percentage);
OnProgressChanged (percentage, speed, information);
last_change = DateTime.Now;
}
}
git_clone.WaitForExit ();
@ -151,78 +142,6 @@ namespace Sparkles.Git {
}
Regex progress_regex = new Regex (@"([0-9]+)%", RegexOptions.Compiled);
Regex progress_regex_lfs = new Regex (@"^Git LFS:.*([0-9]+) of ([0-9]+).*", RegexOptions.Compiled);
Regex speed_regex = new Regex (@"([0-9\.]+) ([KM])iB/s", RegexOptions.Compiled);
public bool ParseProgress (string line, out double percentage, out double speed, out string information)
{
percentage = 0;
speed = 0;
information = "";
Match match;
if (FetchedRepoStorageType == StorageType.LargeFiles) {
match = progress_regex_lfs.Match (line);
if (!match.Success)
return true;
percentage = double.Parse (match.Groups [1].Value) / double.Parse (match.Groups [2].Value) * 100;
information = string.Format ("{0} of {1} files", match.Groups [1].Value, match.Groups [2].Value);
return true;
}
match = progress_regex.Match (line);
if (!match.Success) {
Logger.LogInfo ("Fetcher", line);
line = line.Trim (new char [] { ' ', '@' });
if (line.StartsWith ("fatal:", StringComparison.InvariantCultureIgnoreCase) ||
line.StartsWith ("error:", StringComparison.InvariantCultureIgnoreCase)) {
errors.Add (line);
} else if (line.StartsWith ("WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!")) {
errors.Add ("warning: Remote host identification has changed");
} else if (line.StartsWith ("WARNING: POSSIBLE DNS SPOOFING DETECTED!")) {
errors.Add ("warning: Possible DNS spoofing detected");
}
return false;
}
int number = int.Parse (match.Groups [1].Value, new CultureInfo ("en-US"));
// The pushing progress consists of two stages: the "Compressing
// objects" stage which we count as 20% of the total progress, and
// the "Writing objects" stage which we count as the last 80%
if (line.Contains ("Compressing objects")) {
// "Compressing objects" stage
percentage = (number / 100 * 20);
} else if (line.Contains ("Writing objects")) {
percentage = (number / 100 * 80 + 20);
Match speed_match = speed_regex.Match (line);
if (speed_match.Success) {
speed = double.Parse (speed_match.Groups [1].Value, new CultureInfo ("en-US")) * 1024;
if (speed_match.Groups [2].Value.Equals ("M"))
speed = speed * 1024;
information = speed.ToSize ();
}
}
return true;
}
public override void Stop ()
{
try {

205
Sparkles/Git/GitRepository.cs
View file

@ -17,7 +17,6 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
@ -224,61 +223,8 @@ namespace Sparkles.Git {
git_push.StartInfo.RedirectStandardError = true;
git_push.Start ();
double percentage = 1.0;
// TODO: parse LFS progress
while (!git_push.StandardError.EndOfStream) {
string line = git_push.StandardError.ReadLine ();
Console.WriteLine (line);
Match match = this.progress_regex.Match (line);
double speed = 0.0;
double number = 0.0;
if (match.Success) {
try {
number = double.Parse (match.Groups [1].Value, new CultureInfo ("en-US"));
} catch (FormatException) {
Logger.LogInfo ("Git", "Error parsing progress: \"" + match.Groups [1] + "\"");
}
// The pushing progress consists of two stages: the "Compressing
// objects" stage which we count as 20% of the total progress, and
// the "Writing objects" stage which we count as the last 80%
if (line.StartsWith ("Compressing")) {
// "Compressing objects" stage
number = (number / 100 * 20);
} else {
// "Writing objects" stage
number = (number / 100 * 80 + 20);
Match speed_match = this.speed_regex.Match (line);
if (speed_match.Success) {
try {
speed = double.Parse (speed_match.Groups [1].Value, new CultureInfo ("en-US")) * 1024;
} catch (FormatException) {
Logger.LogInfo ("Git", "Error parsing speed: \"" + speed_match.Groups [1] + "\"");
}
if (speed_match.Groups [2].Value.Equals ("M"))
speed = speed * 1024;
}
}
} else {
Logger.LogInfo ("Git", Name + " | " + line);
if (FindError (line))
return false;
}
if (number >= percentage) {
percentage = number;
base.OnProgressChanged (percentage, speed);
}
}
if (!ReadStream (git_push))
return false;
git_push.WaitForExit ();
@ -299,70 +245,17 @@ namespace Sparkles.Git {
if (StorageType == StorageType.LargeFiles)
File.Create (lfs_is_behind_file_path);
var git = new GitCommand (LocalPath, "fetch --progress origin " + branch, auth_info);
var git_fetch = new GitCommand (LocalPath, "fetch --progress origin " + branch, auth_info);
git.StartInfo.RedirectStandardError = true;
git.Start ();
git_fetch.StartInfo.RedirectStandardError = true;
git_fetch.Start ();
double percentage = 1.0;
if (!ReadStream (git_fetch))
return false;
// TODO: parse LFS progress
while (!git.StandardError.EndOfStream) {
string line = git.StandardError.ReadLine ();
Match match = this.progress_regex.Match (line);
double speed = 0.0;
double number = 0.0;
git_fetch.WaitForExit ();
if (match.Success) {
try {
number = double.Parse (match.Groups [1].Value, new CultureInfo ("en-US"));
} catch (FormatException) {
Logger.LogInfo ("Git", "Error parsing progress: \"" + match.Groups [1] + "\"");
}
// The fetching progress consists of two stages: the "Compressing
// objects" stage which we count as 20% of the total progress, and
// the "Receiving objects" stage which we count as the last 80%
if (line.StartsWith ("Compressing")) {
// "Compressing objects" stage
number = (number / 100 * 20);
} else {
// "Writing objects" stage
number = (number / 100 * 80 + 20);
Match speed_match = this.speed_regex.Match (line);
if (speed_match.Success) {
try {
speed = double.Parse (speed_match.Groups [1].Value, new CultureInfo ("en-US")) * 1024;
} catch (FormatException) {
Logger.LogInfo ("Git", "Error parsing speed: \"" + speed_match.Groups [1] + "\"");
}
if (speed_match.Groups [2].Value.Equals ("M"))
speed = speed * 1024;
}
}
} else {
Logger.LogInfo ("Git", Name + " | " + line);
if (FindError (line))
return false;
}
if (number >= percentage) {
percentage = number;
base.OnProgressChanged (percentage, speed);
}
}
git.WaitForExit ();
if (git.ExitCode != 0) {
if (git_fetch.ExitCode != 0) {
Error = ErrorStatus.HostUnreachable;
return false;
}
@ -390,6 +283,45 @@ namespace Sparkles.Git {
}
bool ReadStream (GitCommand command)
{
StreamReader output_stream = command.StandardError;
if (StorageType == StorageType.LargeFiles)
output_stream = command.StandardOutput;
double previous_percentage = 0;
double percentage = 0;
double speed = 0;
string information = "";
while (!output_stream.EndOfStream) {
string line = output_stream.ReadLine ();
previous_percentage = percentage;
ErrorStatus error = GitCommand.ParseProgress (line, out percentage, out speed, out information);
if (error != ErrorStatus.None) {
Error = error;
information = line;
command.Kill ();
command.Dispose ();
Logger.LogInfo ("Git", Name + " | Error status changed to " + Error);
return false;
}
if (percentage <= previous_percentage)
continue;
OnProgressChanged (percentage, speed, information);
}
return true;
}
public override bool HasLocalChanges {
get {
PrepareDirectories (LocalPath);
@ -685,44 +617,6 @@ namespace Sparkles.Git {
}
bool FindError (string line)
{
Error = ErrorStatus.None;
if (line.Contains ("WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!") ||
line.Contains ("WARNING: POSSIBLE DNS SPOOFING DETECTED!")) {
Error = ErrorStatus.HostIdentityChanged;
} else if (line.StartsWith ("Permission denied") ||
line.StartsWith ("ssh_exchange_identification: Connection closed by remote host") ||
line.StartsWith ("The authenticity of host")) {
Error = ErrorStatus.AuthenticationFailed;
} else if (line.EndsWith ("does not appear to be a git repository")) {
Error = ErrorStatus.NotFound;
} else if (line.EndsWith ("expected old/new/ref, got 'shallow")) {
Error = ErrorStatus.IncompatibleClientServer;
} else if (line.StartsWith ("error: Disk space exceeded") ||
line.EndsWith ("No space left on device") ||
line.EndsWith ("file write error (Disk quota exceeded)")) {
Error = ErrorStatus.DiskSpaceExceeded;
}
if (Error != ErrorStatus.None) {
Logger.LogInfo ("Git", Name + " | Error status changed to " + Error);
return true;
} else {
return false;
}
}
public override List<Change> UnsyncedChanges {
get {
return ParseStatus ();
@ -1176,9 +1070,6 @@ namespace Sparkles.Git {
}
Regex progress_regex = new Regex (@"([0-9]+)%", RegexOptions.Compiled);
Regex speed_regex = new Regex (@"([0-9\.]+) ([KM])iB/s", RegexOptions.Compiled);
Regex log_regex = new Regex (@"commit ([a-f0-9]{40})*\n" +
"Author: (.+) <(.+)>\n" +
"Date: ([0-9]{4})-([0-9]{2})-([0-9]{2}) " +