Progress on LFS support

This commit is contained in:
Hylke Bons 2016-06-17 17:03:40 -07:00 committed by Hylke Bons
parent 6376a5518a
commit 1e37ba0992
8 changed files with 254 additions and 162 deletions

View file

@ -480,9 +480,8 @@ namespace SparkleShare {
void StartupInviteScan ()
{
foreach (string invite in Directory.GetFiles (FoldersPath, "*.xml")) {
foreach (string invite in Directory.GetFiles (FoldersPath, "*.xml"))
HandleInvite (invite);
}
}
@ -548,7 +547,14 @@ namespace SparkleShare {
OnIdle ();
}
public List<StorageTypeInfo> FetcherAvailableStorageTypes {
get {
return this.fetcher.AvailableStorageTypes;
}
}
public void StartFetcher (SparkleFetcherInfo info)
{
string tmp_path = Config.TmpPath;
@ -573,7 +579,7 @@ namespace SparkleShare {
} catch (Exception e) {
Logger.LogInfo ("Controller",
"Failed to load '" + backend + "' backend for '" + canonical_name + "' " + e.Message);
"Failed to load '" + backend + "' backend for '" + canonical_name + "' " + e.Message);
FolderFetchError (Path.Combine (info.Address, info.RemotePath).Replace (@"\", "/"),
new string [] {"Failed to load \"" + backend + "\" backend for \"" + canonical_name + "\""});
@ -581,38 +587,43 @@ namespace SparkleShare {
return;
}
this.fetcher.Finished += delegate (bool repo_is_encrypted, bool repo_is_empty, string [] warnings) {
this.fetcher.Finished += FetcherFinishedDelegate;
this.fetcher.Failed += FetcherFailedDelegate;
this.fetcher.ProgressChanged += FetcherProgressChangedDelgate;
if (repo_is_empty) {
ShowSetupWindow (PageType.StorageSetup);
}
return; // TODO
if (repo_is_encrypted && repo_is_empty) {
ShowSetupWindowEvent (PageType.CryptoSetup);
} else if (repo_is_encrypted) {
ShowSetupWindowEvent (PageType.CryptoPassword);
} else {
FinishFetcher ();
}
};
this.fetcher.Failed += delegate {
FolderFetchError (this.fetcher.RemoteUrl.ToString (), this.fetcher.Errors);
StopFetcher ();
};
this.fetcher.ProgressChanged += delegate (double percentage, double speed) {
FolderFetching (percentage, speed);
};
this.fetcher.Start ();
}
void FetcherFinishedDelegate (StorageType storage_type, string [] warnings)
{
if (storage_type == StorageType.Unknown) {
ShowSetupWindow (PageType.StorageSetup);
return;
}
if (storage_type == StorageType.Encrypted) {
ShowSetupWindowEvent (PageType.CryptoPassword);
return;
}
FinishFetcher ();
}
void FetcherFailedDelegate ()
{
FolderFetchError (this.fetcher.RemoteUrl.ToString (), this.fetcher.Errors);
StopFetcher ();
}
void FetcherProgressChangedDelgate (double percentage, double speed)
{
FolderFetching (percentage, speed);
}
public void StopFetcher ()
{
this.fetcher.Stop ();
@ -629,65 +640,27 @@ namespace SparkleShare {
}
public void FinishFetcher (StorageType storage_type)
{
if (storage_type == StorageType.Media) {
FinishFetcher (); // TODO: enable large files
return;
}
FinishFetcher ();
}
public void FinishFetcher (StorageType storage_type, string password)
{
if (storage_type != StorageType.Encrypted)
return;
this.fetcher.EnableFetchedRepoCrypto (password);
FinishFetcher ();
}
public void FinishFetcher ()
{
FinishFetcher (StorageType.Plain);
}
public void FinishFetcher (StorageType storage_type, string password)
{
this.fetcher.EnableFetchedRepoCrypto (password);
FinishFetcher (StorageType.Encrypted);
}
public void FinishFetcher (StorageType storage_type)
{
this.watcher.EnableRaisingEvents = false;
this.fetcher.Complete ();
string canonical_name = Path.GetFileName (this.fetcher.RemoteUrl.AbsolutePath);
if (canonical_name.EndsWith (".git"))
canonical_name = canonical_name.Replace (".git", "");
canonical_name = canonical_name.Replace ("-crypto", "");
canonical_name = canonical_name.ReplaceUnderscoreWithSpace ();
canonical_name = canonical_name.Replace ("%20", " ");
bool target_folder_exists = Directory.Exists (
Path.Combine (Config.FoldersPath, canonical_name));
// Add a numbered suffix to the name if a folder with the same name
// already exists. Example: "Folder (2)"
int suffix = 1;
while (target_folder_exists) {
suffix++;
target_folder_exists = Directory.Exists (
Path.Combine (Config.FoldersPath, canonical_name + " (" + suffix + ")"));
}
string target_folder_name = canonical_name;
this.fetcher.Complete (storage_type);
if (suffix > 1)
target_folder_name += " (" + suffix + ")";
string target_folder_path = DetermineFolderPath ();
string target_folder_name = Path.GetFileName (target_folder_path);
string group_folder_path = Path.Combine (Config.FoldersPath, this.fetcher.RemoteUrl.Host);
if (!Directory.Exists (group_folder_path))
Directory.CreateDirectory (group_folder_path);
string target_folder_path = Path.Combine (group_folder_path, target_folder_name);
try {
Directory.Move (this.fetcher.TargetFolder, target_folder_path);
@ -711,24 +684,48 @@ namespace SparkleShare {
string backend = BaseFetcher.GetBackend (this.fetcher.RemoteUrl.ToString ());
Config.AddFolder (target_folder_name, this.fetcher.Identifier,
this.fetcher.RemoteUrl.ToString (), backend);
this.fetcher.RemoteUrl.ToString (), backend);
if (storage_type != StorageType.Plain) {
Config.SetFolderOptionalAttribute (target_folder_name,
"storage_type", storage_type.ToString ());
}
if (this.fetcher.OriginalFetcherInfo.AnnouncementsUrl != null) {
Config.SetFolderOptionalAttribute (target_folder_name, "announcements_url",
this.fetcher.OriginalFetcherInfo.AnnouncementsUrl);
this.fetcher.OriginalFetcherInfo.AnnouncementsUrl);
}
RepositoriesLoaded = true;
FolderFetched (this.fetcher.RemoteUrl.ToString (), this.fetcher.Warnings.ToArray ());
AddRepository (target_folder_path);
RepositoriesLoaded = true;
FolderListChanged ();
FolderFetched (this.fetcher.RemoteUrl.ToString (), this.fetcher.Warnings.ToArray ());
this.fetcher.Dispose ();
this.fetcher = null;
this.watcher.EnableRaisingEvents = true;
}
string DetermineFolderPath ()
{
string folder_name = this.fetcher.FormatName ();
string folder_group_path = Path.Combine (Config.FoldersPath, this.fetcher.RemoteUrl.Host);
string folder_path = Path.Combine (Config.FoldersPath, folder_group_path, folder_name);
if (!Directory.Exists (folder_path)) {
if (!Directory.Exists (folder_group_path))
Directory.CreateDirectory (folder_group_path);
return folder_path;
}
// Add a number suffix when needed, e.g. "Folder (3)"
int suffix = 2 + Directory.GetDirectories (folder_group_path, folder_name + " (*").Length;
return string.Format ("{0} ({1})", folder_path, suffix);
}
public virtual void Quit ()

View file

@ -23,13 +23,16 @@ using System.Threading;
namespace Sparkles {
public class SparkleFetcherInfo {
public string Address;
public string Address; // TODO: Uri object
public string RemotePath;
public string Backend;
public string Fingerprint;
public string Backend;
public string TargetDirectory;
public string AnnouncementsUrl;
public bool FetchPriorHistory;
public string AnnouncementsUrl; // TODO: Uri object
}
@ -39,7 +42,7 @@ namespace Sparkles {
public event Action Failed = delegate { };
public event FinishedEventHandler Finished = delegate { };
public delegate void FinishedEventHandler (bool repo_is_encrypted, bool repo_is_empty, string [] warnings);
public delegate void FinishedEventHandler (StorageType storage_type, string [] warnings);
public event ProgressChangedEventHandler ProgressChanged = delegate { };
public delegate void ProgressChangedEventHandler (double percentage, double speed);
@ -47,23 +50,30 @@ namespace Sparkles {
public abstract bool Fetch ();
public abstract void Stop ();
public abstract bool IsFetchedRepoEmpty { get; }
public bool IsActive { get; protected set; }
public double ProgressPercentage { get; private set; }
public double ProgressSpeed { get; private set; }
protected abstract bool IsFetchedRepoEmpty { get; }
protected StorageType FetchedRepoStorageType = StorageType.Unknown;
public abstract bool IsFetchedRepoPasswordCorrect (string password);
public abstract void EnableFetchedRepoCrypto (string password);
protected readonly List<StorageTypeInfo> AvailableStorageTypes = new List<StorageTypeInfo> ();
public readonly List<StorageTypeInfo> AvailableStorageTypes = new List<StorageTypeInfo> ();
public double ProgressPercentage { get; private set; }
public double ProgressSpeed { get; private set; }
public Uri RemoteUrl { get; protected set; }
public string RequiredFingerprint { get; protected set; }
public readonly bool FetchPriorHistory;
public string TargetFolder { get; protected set; }
public bool IsActive { get; protected set; }
public string Identifier;
public SparkleFetcherInfo OriginalFetcherInfo;
protected List<string> warnings = new List<string> ();
protected List<string> errors = new List<string> ();
public string [] Warnings {
get {
return warnings.ToArray ();
@ -77,18 +87,11 @@ namespace Sparkles {
}
protected List<string> warnings = new List<string> ();
protected List<string> errors = new List<string> ();
Thread thread;
protected BaseFetcher (SparkleFetcherInfo info)
{
AvailableStorageTypes.Add (
new StorageTypeInfo (StorageType.Plain, "Plain Storage", "Nothing fancy."));
new StorageTypeInfo (StorageType.Plain, "Plain Storage", "Nothing fancy"));
OriginalFetcherInfo = info;
RequiredFingerprint = info.Fingerprint;
@ -112,6 +115,8 @@ namespace Sparkles {
}
Thread thread;
public void Start ()
{
IsActive = true;
@ -136,9 +141,7 @@ namespace Sparkles {
Logger.LogInfo ("Fetcher", "Finished");
IsActive = false;
bool repo_is_encrypted = RemoteUrl.AbsolutePath.Contains ("-crypto");
Finished (repo_is_encrypted, IsFetchedRepoEmpty, Warnings);
Finished (FetchedRepoStorageType, Warnings);
} else {
Thread.Sleep (500);
@ -159,7 +162,18 @@ namespace Sparkles {
}
public virtual void Complete ()
public void Complete ()
{
if (FetchedRepoStorageType == StorageType.Unknown) {
Complete (StorageType.Plain);
return;
}
this.Complete (FetchedRepoStorageType);
}
public virtual void Complete (StorageType storage_type)
{
string identifier_path = Path.Combine (TargetFolder, ".sparkleshare");
@ -203,7 +217,7 @@ namespace Sparkles {
n +
"Have fun! :)" + n;
if (RemoteUrl.AbsolutePath.Contains ("-crypto") || RemoteUrl.Host.Equals ("sparkleshare.net"))
if (FetchedRepoStorageType == StorageType.Encrypted)
text = text.Replace ("a SparkleShare repository", "an encrypted SparkleShare repository");
File.WriteAllText (file_path, text);
@ -216,13 +230,6 @@ namespace Sparkles {
}
public void Dispose ()
{
if (thread != null)
thread.Abort ();
}
protected void OnProgressChanged (double percentage, double speed) {
ProgressChanged (percentage, speed);
}
@ -241,6 +248,19 @@ namespace Sparkles {
}
public virtual string FormatName ()
{
return Path.GetFileName (RemoteUrl.AbsolutePath);
}
public void Dispose ()
{
if (thread != null)
thread.Abort ();
}
protected string [] ExcludeRules = {
"*.autosave", // Various autosaving apps
"*~", // gedit and emacs

View file

@ -191,6 +191,11 @@ namespace Sparkles {
this.identifier = Identifier;
ChangeSets = GetChangeSets ();
string storage_type = this.local_config.GetFolderOptionalAttribute (Name, "storage_type");
if (!string.IsNullOrEmpty (storage_type))
StorageType = (StorageType) Enum.Parse(typeof(StorageType), storage_type);
string is_paused = this.local_config.GetFolderOptionalAttribute (Name, "paused");
if (is_paused != null && is_paused.Equals (bool.TrueString))
Status = SyncStatus.Paused;

View file

@ -27,7 +27,7 @@ namespace Sparkles {
bool write_output;
public Command (string path, string args) : this (path, args, false)
public Command (string path, string args) : this (path, args, true)
{
}

View file

@ -95,12 +95,12 @@ namespace Sparkles {
} else if (day_diff < 31) {
if (day_diff < 14)
return "last week";
return "a week ago";
else
return string.Format ("{0} weeks ago", Math.Ceiling ((double) day_diff / 7));
} else if (day_diff < 62) {
return "last month";
return "a month ago";
} else {
return string.Format ("{0} months ago", Math.Ceiling ((double) day_diff / 31));

View file

@ -45,7 +45,7 @@ namespace Sparkles.Git {
if (GitPath == null)
GitPath = LocateCommand ("git");
string git_version = new Command (GitPath, "--version").StartAndReadStandardOutput ();
string git_version = new Command (GitPath, "--version", false).StartAndReadStandardOutput ();
return git_version.Replace ("git version ", "");
}
}
@ -56,7 +56,7 @@ namespace Sparkles.Git {
if (GitPath == null)
GitPath = LocateCommand ("git");
string git_lfs_version = new Command (GitPath, "lfs version").StartAndReadStandardOutput ();
string git_lfs_version = new Command (GitPath, "lfs version", false).StartAndReadStandardOutput ();
return git_lfs_version.Replace ("git-lfs/", "").Split (' ') [0];
}
}

View file

@ -34,7 +34,7 @@ namespace Sparkles.Git {
string password_salt = "662282447f6bbb8c8e15fb32dd09e3e708c32bc8";
public override bool IsFetchedRepoEmpty {
protected override bool IsFetchedRepoEmpty {
get {
var git_rev_parse = new GitCommand (TargetFolder, "rev-parse HEAD");
git_rev_parse.StartAndWaitForExit ();
@ -46,10 +46,6 @@ namespace Sparkles.Git {
public GitFetcher (SparkleFetcherInfo fetcher_info, SSHAuthenticationInfo auth_info) : base (fetcher_info)
{
AvailableStorageTypes.Add (
new StorageTypeInfo (StorageType.Encrypted, "Encrypted Storage",
"Trade off efficiency for privacy; encrypt files before storing them."));
this.auth_info = auth_info;
var uri_builder = new UriBuilder (RemoteUrl);
@ -60,8 +56,8 @@ namespace Sparkles.Git {
RemoteUrl.Host.Equals ("gitlab.com")) {
AvailableStorageTypes.Add (
new StorageTypeInfo (StorageType.Media, "Media Storage",
"Trade off versioning for space; don't keep a history locally."));
new StorageTypeInfo (StorageType.Media, "Large File Storage",
"Trade off versioning for space;\ndoesn't keep a local history"));
uri_builder.Scheme = "ssh";
uri_builder.UserName = "git";
@ -74,6 +70,35 @@ namespace Sparkles.Git {
}
RemoteUrl = uri_builder.Uri;
AvailableStorageTypes.Add (
new StorageTypeInfo (StorageType.Encrypted, "Encrypted Storage",
"Trade off efficiency for privacy;\nencrypts before storing files on the host"));
}
StorageType? DetermineStorageType ()
{
var git_ls_remote = new GitCommand (Configuration.DefaultConfiguration.TmpPath,
string.Format ("ls-remote --heads \"{0}\"", RemoteUrl), auth_info);
string output = git_ls_remote.StartAndReadStandardOutput ();
if (git_ls_remote.ExitCode != 0)
return null;
if (string.IsNullOrWhiteSpace (output))
return StorageType.Unknown;
foreach (string line in output.Split ("\n".ToCharArray ())) {
if (line.Contains ("x-sparkleshare-lfs"))
return StorageType.Media;
if (line.Contains ("x-sparkleshare-encrypted"))
return StorageType.Encrypted;
}
return StorageType.Plain;
}
@ -82,14 +107,24 @@ namespace Sparkles.Git {
if (!base.Fetch ())
return false;
if (FetchPriorHistory) {
git_clone = new GitCommand (Configuration.DefaultConfiguration.TmpPath,
"clone --progress --no-checkout \"" + RemoteUrl + "\" \"" + TargetFolder + "\"", auth_info);
StorageType? storage_type = DetermineStorageType ();
} else {
git_clone = new GitCommand (Configuration.DefaultConfiguration.TmpPath,
"clone --progress --no-checkout --depth=1 \"" + RemoteUrl + "\" \"" + TargetFolder + "\"", auth_info);
}
if (storage_type == null)
return false;
FetchedRepoStorageType = (StorageType) storage_type;
string git_clone_command = "clone --progress --no-checkout";
if (FetchPriorHistory)
git_clone_command += " --depth=1";
if (storage_type == StorageType.Media)
git_clone_command = "lfs " + git_clone_command;
var git_clone = new GitCommand (Configuration.DefaultConfiguration.TmpPath,
string.Format ("{0} \"{1}\" \"{2}\"", git_clone_command, RemoteUrl, TargetFolder),
auth_info);
git_clone.StartInfo.RedirectStandardError = true;
git_clone.Start ();
@ -192,7 +227,6 @@ namespace Sparkles.Git {
InstallExcludeRules ();
InstallAttributeRules ();
InstallGitLFS ();
EnableGitLFS (); // TODO
return true;
}
@ -222,14 +256,36 @@ namespace Sparkles.Git {
}
public override void Complete ()
public override void Complete (StorageType selected_storage_type)
{
if (!IsFetchedRepoEmpty) {
if (IsFetchedRepoEmpty) {
var git_commit = new GitCommand (TargetFolder,
"commit --allow-empty --mesage=\"Initial commit by SparkleShare\"");
git_commit.StartAndWaitForExit ();
// These branches will be pushed later by "git push --all"
if (selected_storage_type == StorageType.Media) {
var git_branch = new GitCommand (TargetFolder, "branch x-sparkleshare-lfs", auth_info);
git_branch.StartAndWaitForExit ();
InstallGitLFS ();
EnableGitLFS ();
}
if (selected_storage_type == StorageType.Encrypted) {
var git_branch = new GitCommand (TargetFolder, "branch x-sparkleshare-encrypted", auth_info);
git_branch.StartAndWaitForExit ();
}
} else {
string branch = "HEAD";
string prefered_branch = "SparkleShare";
// Prefer the "SparkleShare" branch if it exists
var git_show_ref = new GitCommand (TargetFolder, "show-ref --verify --quiet refs/heads/" + prefered_branch);
var git_show_ref = new GitCommand (TargetFolder,
"show-ref --verify --quiet refs/heads/" + prefered_branch);
git_show_ref.StartAndWaitForExit ();
if (git_show_ref.ExitCode == 0)
@ -239,7 +295,7 @@ namespace Sparkles.Git {
git_checkout.StartAndWaitForExit ();
}
base.Complete ();
base.Complete (selected_storage_type);
}
@ -331,6 +387,18 @@ namespace Sparkles.Git {
}
public override string FormatName ()
{
string name = Path.GetFileName (RemoteUrl.AbsolutePath);
name = name.ReplaceUnderscoreWithSpace ();
if (name.EndsWith (".git"))
name = name.Replace (".git", "");
return name;
}
void InstallExcludeRules ()
{
string git_info_path = Path.Combine (TargetFolder, ".git", "info");
@ -373,13 +441,15 @@ namespace Sparkles.Git {
void InstallGitLFS ()
{
var git_config_required = new GitCommand (TargetFolder, "config filter.lfs.required true");
var git_config_clean = new GitCommand (TargetFolder, "config filter.lfs.clean 'git-lfs clean %f'");
string GIT_SSH_COMMAND = GitCommand.FormatGitSSHCommand (auth_info);
string smudge_command = "env GIT_SSH_COMMAND='" + GIT_SSH_COMMAND + "' git-lfs smudge %f";
var git_config_smudge = new GitCommand (TargetFolder,
"config filter.lfs.smudge \"" + smudge_command + "\"");
var git_config_clean = new GitCommand (TargetFolder,
"config filter.lfs.clean 'git-lfs clean %f'");
git_config_required.StartAndWaitForExit ();
git_config_clean.StartAndWaitForExit ();

View file

@ -57,6 +57,7 @@ namespace Sparkles.Git {
var git = new GitCommand (LocalPath, "config core.ignorecase true");
git.StartAndWaitForExit ();
// TODO: ugly
while (this.in_merge && HasLocalChanges) {
try {
ResolveConflict ();
@ -223,10 +224,12 @@ namespace Sparkles.Git {
string pre_push_hook_path = Path.Combine (LocalPath, ".git", "hooks", "pre-push");
// We start "git lfs push" manually, so remove the
// hook automatically created by the clean/smudge filters
if (File.Exists (pre_push_hook_path))
File.Delete (pre_push_hook_path);
string pre_push_hook_content =
"#!/bin/sh" + Environment.NewLine +
"env GIT_SSH_COMMAND='" + GitCommand.FormatGitSSHCommand (auth_info) + "' " +
"git-lfs pre-push \"$@\"";
File.WriteAllText (pre_push_hook_path, pre_push_hook_content);
if (StorageType == StorageType.Media) {
// TODO: Progress reporting, error handling
@ -234,12 +237,13 @@ namespace Sparkles.Git {
git_lfs_push.StartAndWaitForExit ();
}
var git_push = new GitCommand (LocalPath, "push --progress \"" + RemoteUrl + "\" " + this.branch, auth_info);
var git_push = new GitCommand (LocalPath, string.Format ("push --all --progress \"{0}\"", RemoteUrl), auth_info);
git_push.StartInfo.RedirectStandardError = true;
git_push.Start ();
double percentage = 1.0;
// TODO: parse LFS progress
while (!git_push.StandardError.EndOfStream) {
string line = git_push.StandardError.ReadLine ();
Match match = this.progress_regex.Match (line);
@ -305,18 +309,14 @@ namespace Sparkles.Git {
public override bool SyncDown ()
{
if (StorageType == StorageType.Media) {
var git_lfs_pull = new GitCommand (LocalPath, "lfs pull", auth_info);
git_lfs_pull.StartAndWaitForExit ();
}
var git = new GitCommand (LocalPath, "fetch --progress \"" + RemoteUrl + "\" " + this.branch, auth_info);
var git = new GitCommand (LocalPath, "fetch --progress \"" + RemoteUrl + "\" " + branch, auth_info);
git.StartInfo.RedirectStandardError = true;
git.Start ();
double percentage = 1.0;
// TODO: parse LFS progress
while (!git.StandardError.EndOfStream) {
string line = git.StandardError.ReadLine ();
Match match = this.progress_regex.Match (line);
@ -751,20 +751,20 @@ namespace Sparkles.Git {
GitCommand git;
if (path == null) {
git = new GitCommand (LocalPath, "log --since=1.month --raw --find-renames --date=iso " +
git = new GitCommand (LocalPath, "--no-pager log --since=1.month --raw --find-renames --date=iso " +
"--format=medium --no-color --no-merges");
} else {
path = path.Replace ("\\", "/");
git = new GitCommand (LocalPath, "log --raw --find-renames --date=iso " +
git = new GitCommand (LocalPath, "--no-pager log --raw --find-renames --date=iso " +
"--format=medium --no-color --no-merges -- \"" + path + "\"");
}
string output = git.StartAndReadStandardOutput ();
if (path == null && string.IsNullOrWhiteSpace (output)) {
git = new GitCommand (LocalPath, "log -n 75 --raw --find-renames --date=iso " +
git = new GitCommand (LocalPath, "--no-pager log -n 75 --raw --find-renames --date=iso " +
"--format=medium --no-color --no-merges");
output = git.StartAndReadStandardOutput ();