Make Git LFS work properly

This commit is contained in:
Hylke Bons 2016-06-20 13:19:20 -07:00 committed by Hylke Bons
parent 3ca4094224
commit 6716f1dcce
6 changed files with 70 additions and 57 deletions

View file

@ -536,6 +536,7 @@ namespace SparkleShare {
string folder_name = Path.GetFileName (PreviousPath); string folder_name = Path.GetFileName (PreviousPath);
folder_name = folder_name.ReplaceUnderscoreWithSpace (); folder_name = folder_name.ReplaceUnderscoreWithSpace ();
// TODO: Open SparkleShare/$HOST
SparkleShare.Controller.OpenSparkleShareFolder (folder_name); SparkleShare.Controller.OpenSparkleShareFolder (folder_name);
FinishPageCompleted (); FinishPageCompleted ();
} }

View file

@ -56,7 +56,7 @@ namespace Sparkles {
protected abstract bool IsFetchedRepoEmpty { get; } protected abstract bool IsFetchedRepoEmpty { get; }
protected StorageType FetchedRepoStorageType = StorageType.Unknown; public StorageType FetchedRepoStorageType { get; protected set; }
public abstract bool IsFetchedRepoPasswordCorrect (string password); public abstract bool IsFetchedRepoPasswordCorrect (string password);
public abstract void EnableFetchedRepoCrypto (string password); public abstract void EnableFetchedRepoCrypto (string password);
@ -89,6 +89,8 @@ namespace Sparkles {
protected BaseFetcher (SparkleFetcherInfo info) protected BaseFetcher (SparkleFetcherInfo info)
{ {
FetchedRepoStorageType = StorageType.Unknown;
AvailableStorageTypes.Add ( AvailableStorageTypes.Add (
new StorageTypeInfo (StorageType.Plain, "Plain Storage", "Nothing fancy")); new StorageTypeInfo (StorageType.Plain, "Plain Storage", "Nothing fancy"));

View file

@ -27,7 +27,7 @@ namespace Sparkles {
public enum StorageType { public enum StorageType {
Unknown, Unknown,
Plain, Plain,
Media, LargeFiles,
Encrypted Encrypted
} }

View file

@ -20,7 +20,7 @@ namespace Sparkles.Git {
public class GitCommand : Command { public class GitCommand : Command {
public static string SSHPath; public static string SSHPath = "ssh";
public static string ExecPath; public static string ExecPath;
@ -71,9 +71,6 @@ namespace Sparkles.Git {
{ {
StartInfo.WorkingDirectory = working_dir; StartInfo.WorkingDirectory = working_dir;
if (string.IsNullOrEmpty (SSHPath))
SSHPath = "ssh";
string GIT_SSH_COMMAND = SSHPath; string GIT_SSH_COMMAND = SSHPath;
if (auth_info != null) if (auth_info != null)

View file

@ -57,7 +57,7 @@ namespace Sparkles.Git {
RemoteUrl.Host.Equals ("gitlab.com")) { RemoteUrl.Host.Equals ("gitlab.com")) {
AvailableStorageTypes.Add ( AvailableStorageTypes.Add (
new StorageTypeInfo (StorageType.Media, "Large File Storage", new StorageTypeInfo (StorageType.LargeFiles, "Large File Storage",
"Trade off versioning for space;\ndoesn't keep a local history")); "Trade off versioning for space;\ndoesn't keep a local history"));
uri_builder.Scheme = "ssh"; uri_builder.Scheme = "ssh";
@ -96,7 +96,7 @@ namespace Sparkles.Git {
string branch = line_parts [line_parts.Length - 1]; string branch = line_parts [line_parts.Length - 1];
if (branch == "x-sparkleshare-lfs") if (branch == "x-sparkleshare-lfs")
return StorageType.Media; return StorageType.LargeFiles;
string encrypted_storage_prefix = "x-sparkleshare-encrypted-"; string encrypted_storage_prefix = "x-sparkleshare-encrypted-";
@ -127,7 +127,7 @@ namespace Sparkles.Git {
if (!FetchPriorHistory) if (!FetchPriorHistory)
git_clone_command += " --depth=1"; git_clone_command += " --depth=1";
if (storage_type == StorageType.Media) if (storage_type == StorageType.LargeFiles)
git_clone_command = "lfs clone --progress --no-checkout"; git_clone_command = "lfs clone --progress --no-checkout";
var git_clone = new GitCommand (Configuration.DefaultConfiguration.TmpPath, var git_clone = new GitCommand (Configuration.DefaultConfiguration.TmpPath,
@ -231,11 +231,6 @@ namespace Sparkles.Git {
OnProgressChanged (100, 0); OnProgressChanged (100, 0);
InstallConfiguration ();
InstallExcludeRules ();
InstallAttributeRules ();
InstallGitLFS ();
return true; return true;
} }
@ -266,6 +261,12 @@ namespace Sparkles.Git {
public override string Complete (StorageType selected_storage_type) public override string Complete (StorageType selected_storage_type)
{ {
InstallConfiguration ();
InstallGitLFS ();
InstallAttributeRules ();
InstallExcludeRules ();
string identifier = base.Complete (selected_storage_type); string identifier = base.Complete (selected_storage_type);
string identifier_path = Path.Combine (TargetFolder, ".sparkleshare"); string identifier_path = Path.Combine (TargetFolder, ".sparkleshare");
@ -280,12 +281,9 @@ namespace Sparkles.Git {
git_commit.StartAndWaitForExit (); git_commit.StartAndWaitForExit ();
// These branches will be pushed later by "git push --all" // These branches will be pushed later by "git push --all"
if (selected_storage_type == StorageType.Media) { if (selected_storage_type == StorageType.LargeFiles) {
var git_branch = new GitCommand (TargetFolder, "branch x-sparkleshare-lfs", auth_info); var git_branch = new GitCommand (TargetFolder, "branch x-sparkleshare-lfs", auth_info);
git_branch.StartAndWaitForExit (); git_branch.StartAndWaitForExit ();
InstallGitLFS ();
EnableGitLFS ();
} }
if (selected_storage_type == StorageType.Encrypted) { if (selected_storage_type == StorageType.Encrypted) {
@ -311,13 +309,7 @@ namespace Sparkles.Git {
if (git_show_ref.ExitCode == 0) if (git_show_ref.ExitCode == 0)
branch = prefered_branch; branch = prefered_branch;
GitCommand git_checkout; var git_checkout = new GitCommand (TargetFolder, string.Format ("checkout --quiet --force {0}", branch));
if (FetchedRepoStorageType == StorageType.Media)
git_checkout = new GitCommand (TargetFolder, "lfs checkout");
else
git_checkout = new GitCommand (TargetFolder, "checkout --quiet --force " + branch);
git_checkout.StartAndWaitForExit (); git_checkout.StartAndWaitForExit ();
} }
@ -369,11 +361,6 @@ namespace Sparkles.Git {
git_config_smudge.StartAndWaitForExit (); git_config_smudge.StartAndWaitForExit ();
git_config_clean.StartAndWaitForExit (); git_config_clean.StartAndWaitForExit ();
// Pass all files through the encryption filter
// TODO: diff=encryption merge=encryption -text?
string git_attributes_file_path = Path.Combine (TargetFolder, ".git", "info", "attributes");
File.WriteAllText (git_attributes_file_path, "* filter=encryption -diff merge=binary");
// Store the password, TODO: 600 permissions // Store the password, TODO: 600 permissions
string password_file_path = Path.Combine (TargetFolder, ".git", "info", "encryption_password"); string password_file_path = Path.Combine (TargetFolder, ".git", "info", "encryption_password");
File.WriteAllText (password_file_path, password.SHA256 (password_salt)); File.WriteAllText (password_file_path, password.SHA256 (password_salt));
@ -439,8 +426,19 @@ namespace Sparkles.Git {
void InstallAttributeRules () void InstallAttributeRules ()
{ {
string attribute_rules_file_path = Path.Combine (TargetFolder, ".git", "info", "attributes"); string git_attributes_file_path = Path.Combine (TargetFolder, ".git", "info", "attributes");
TextWriter writer = new StreamWriter (attribute_rules_file_path);
if (FetchedRepoStorageType == StorageType.LargeFiles) {
File.WriteAllText (git_attributes_file_path, "* filter=lfs diff=lfs merge=lfs -text");
return;
}
if (FetchedRepoStorageType == StorageType.Encrypted) {
File.WriteAllText (git_attributes_file_path, "* filter=encryption -diff -delta merge=binary");
return;
}
TextWriter writer = new StreamWriter (git_attributes_file_path);
// Treat all files as binary as we always want to keep both file versions on a conflict // Treat all files as binary as we always want to keep both file versions on a conflict
writer.WriteLine ("* merge=binary"); writer.WriteLine ("* merge=binary");
@ -480,12 +478,5 @@ namespace Sparkles.Git {
git_config_clean.StartAndWaitForExit (); git_config_clean.StartAndWaitForExit ();
git_config_smudge.StartAndWaitForExit (); git_config_smudge.StartAndWaitForExit ();
} }
void EnableGitLFS ()
{
string git_attributes_file_path = Path.Combine (TargetFolder, ".gitattributes");
File.WriteAllText (git_attributes_file_path, "* filter=lfs diff=lfs merge=lfs -text");
}
} }
} }

View file

@ -203,6 +203,7 @@ namespace Sparkles.Git {
string pre_push_hook_path = Path.Combine (LocalPath, ".git", "hooks", "pre-push"); string pre_push_hook_path = Path.Combine (LocalPath, ".git", "hooks", "pre-push");
// The pre-push hook may have been changed by Git LFS, overwrite it to use our own configuration
string pre_push_hook_content = string pre_push_hook_content =
"#!/bin/sh" + Environment.NewLine + "#!/bin/sh" + Environment.NewLine +
"env GIT_SSH_COMMAND='" + GitCommand.FormatGitSSHCommand (auth_info) + "' " + "env GIT_SSH_COMMAND='" + GitCommand.FormatGitSSHCommand (auth_info) + "' " +
@ -210,12 +211,6 @@ namespace Sparkles.Git {
File.WriteAllText (pre_push_hook_path, pre_push_hook_content); File.WriteAllText (pre_push_hook_path, pre_push_hook_content);
if (StorageType == StorageType.Media) {
// TODO: Progress reporting, error handling
var git_lfs_push = new GitCommand (LocalPath, "lfs push origin " + branch, auth_info);
git_lfs_push.StartAndWaitForExit ();
}
var git_push = new GitCommand (LocalPath, string.Format ("push --all --progress origin", RemoteUrl), auth_info); var git_push = new GitCommand (LocalPath, string.Format ("push --all --progress origin", RemoteUrl), auth_info);
git_push.StartInfo.RedirectStandardError = true; git_push.StartInfo.RedirectStandardError = true;
git_push.Start (); git_push.Start ();
@ -224,9 +219,9 @@ namespace Sparkles.Git {
// TODO: parse LFS progress // TODO: parse LFS progress
while (!git_push.StandardError.EndOfStream) { while (!git_push.StandardError.EndOfStream) {
string line = git_push.StandardError.ReadLine (); string line = git_push.StandardError.ReadLine ();
Match match = this.progress_regex.Match (line); Match match = this.progress_regex.Match (line);
double speed = 0.0; double speed = 0.0;
double number = 0.0; double number = 0.0;
if (match.Success) { if (match.Success) {
@ -276,6 +271,7 @@ namespace Sparkles.Git {
} }
git_push.WaitForExit (); git_push.WaitForExit ();
UpdateSizes (); UpdateSizes ();
if (git_push.ExitCode == 0) if (git_push.ExitCode == 0)
@ -288,6 +284,11 @@ namespace Sparkles.Git {
public override bool SyncDown () public override bool SyncDown ()
{ {
string lfs_is_behind_file_path = Path.Combine (LocalPath, ".git", "lfs", "is_behind");
if (StorageType == StorageType.LargeFiles)
File.Create (lfs_is_behind_file_path);
var git = new GitCommand (LocalPath, "fetch --progress origin " + branch, auth_info); var git = new GitCommand (LocalPath, "fetch --progress origin " + branch, auth_info);
git.StartInfo.RedirectStandardError = true; git.StartInfo.RedirectStandardError = true;
@ -350,18 +351,32 @@ namespace Sparkles.Git {
} }
git.WaitForExit (); git.WaitForExit ();
UpdateSizes ();
if (git.ExitCode == 0) { if (git.ExitCode != 0) {
if (Merge ())
return true;
else
return false;
} else {
Error = ErrorStatus.HostUnreachable; Error = ErrorStatus.HostUnreachable;
return false; return false;
} }
if (Merge ()) {
if (StorageType == StorageType.LargeFiles) {
// Pull LFS files manually to benefit from concurrency
var git_lfs_pull = new GitCommand (LocalPath, "lfs pull origin", auth_info);
git_lfs_pull.StartAndWaitForExit ();
if (git_lfs_pull.ExitCode != 0) {
Error = ErrorStatus.HostUnreachable;
return false;
}
if (File.Exists (lfs_is_behind_file_path))
File.Delete (lfs_is_behind_file_path);
}
UpdateSizes ();
return true;
}
return false;
} }
@ -379,6 +394,13 @@ namespace Sparkles.Git {
public override bool HasUnsyncedChanges { public override bool HasUnsyncedChanges {
get { get {
if (StorageType == StorageType.LargeFiles) {
string lfs_is_behind_file_path = Path.Combine (LocalPath, ".git", "lfs", "is_behind");
if (File.Exists (lfs_is_behind_file_path))
return true;
}
string unsynced_file_path = Path.Combine (LocalPath, ".git", "has_unsynced_changes"); string unsynced_file_path = Path.Combine (LocalPath, ".git", "has_unsynced_changes");
return File.Exists (unsynced_file_path); return File.Exists (unsynced_file_path);
} }
@ -994,7 +1016,7 @@ namespace Sparkles.Git {
if (child_path.EndsWith (".git")) { if (child_path.EndsWith (".git")) {
if (child_path.Equals (Path.Combine (LocalPath, ".git"))) if (child_path.Equals (Path.Combine (LocalPath, ".git")))
continue; continue;
string HEAD_file_path = Path.Combine (child_path, "HEAD"); string HEAD_file_path = Path.Combine (child_path, "HEAD");
if (File.Exists (HEAD_file_path)) { if (File.Exists (HEAD_file_path)) {