From b56ebf3c3d90709a53830d09b3486c95183de1d6 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 6 Sep 2023 10:32:32 +0530 Subject: [PATCH 001/183] Scaffold project --- .gitignore | 8 ++++ go.mod | 8 ++++ go.sum | 10 +++++ internal/crypto/crypto.go | 70 ++++++++++++++++++++++++++++++++++ internal/crypto/crypto_test.go | 53 +++++++++++++++++++++++++ 5 files changed, 149 insertions(+) create mode 100644 .gitignore create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/crypto/crypto.go create mode 100644 internal/crypto/crypto_test.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..33f94e490 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +data/** +.DS_Store +Photos.code-workspace +logs/** +.idea/** +.vscode/** +tmp/** +museum.yaml diff --git a/go.mod b/go.mod new file mode 100644 index 000000000..3c0c2065c --- /dev/null +++ b/go.mod @@ -0,0 +1,8 @@ +module cli-go + +go 1.20 + +require ( + github.com/jamesruan/sodium v1.0.14 + golang.org/x/crypto v0.11.0 +) diff --git a/go.sum b/go.sum new file mode 100644 index 000000000..f3366a91a --- /dev/null +++ b/go.sum @@ -0,0 +1,10 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go new file mode 100644 index 000000000..1bd56ae14 --- /dev/null +++ b/internal/crypto/crypto.go @@ -0,0 +1,70 @@ +package crypto + +import ( + "bytes" + "encoding/base64" + "fmt" + "io" + "log" + + "github.com/jamesruan/sodium" + "golang.org/x/crypto/argon2" +) + +// deriveArgonKey generates a 32-bit cryptographic key using the Argon2id algorithm. +// Parameters: +// - password: The plaintext password to be hashed. +// - salt: The salt as a base64 encoded string. +// - memLimit: The memory limit in bytes. +// - opsLimit: The number of iterations. +// +// Returns: +// - A byte slice representing the derived key. +// - An error object, which is nil if no error occurs. +func deriveArgonKey(password, salt string, memLimit, opsLimit int) ([]byte, error) { + if memLimit < 1024 || opsLimit < 1 { + return nil, fmt.Errorf("invalid memory or operation limits") + } + + // Decode salt from base64 + saltBytes, err := base64.StdEncoding.DecodeString(salt) + if err != nil { + return nil, fmt.Errorf("invalid salt: %v", err) + } + + // Generate key using Argon2id + // Note: We're assuming a fixed key length of 32 bytes and changing the threads + key := argon2.IDKey([]byte(password), saltBytes, uint32(opsLimit), uint32(memLimit/1024), 1, 32) + + return key, nil +} + +// decryptChaCha20poly1305 decrypts the given data using the ChaCha20-Poly1305 algorithm. +// Parameters: +// - data: The encrypted data as a byte slice. +// - key: The key for decryption as a byte slice. +// - nonce: The nonce for decryption as a byte slice. +// +// Returns: +// - A byte slice representing the decrypted data. +// - An error object, which is nil if no error occurs. +func decryptChaCha20poly1305(data []byte, key []byte, nonce []byte) ([]byte, error) { + reader := bytes.NewReader(data) + header := sodium.SecretStreamXCPHeader{Bytes: nonce} + decoder, err := sodium.MakeSecretStreamXCPDecoder( + sodium.SecretStreamXCPKey{Bytes: key}, + reader, + header) + if err != nil { + log.Println("Failed to make secret stream decoder", err) + return nil, err + } + // Buffer to store the decrypted data + decryptedData := make([]byte, len(data)) + n, err := decoder.Read(decryptedData) + if err != nil && err != io.EOF { + log.Println("Failed to read from decoder", err) + return nil, err + } + return decryptedData[:n], nil +} diff --git a/internal/crypto/crypto_test.go b/internal/crypto/crypto_test.go new file mode 100644 index 000000000..ac59b6bbe --- /dev/null +++ b/internal/crypto/crypto_test.go @@ -0,0 +1,53 @@ +package crypto + +import ( + "encoding/base64" + "testing" +) + +const ( + password = "test_password" + kdfSalt = "vd0dcYMGNLKn/gpT6uTFTw==" + memLimit = 64 * 1024 * 1024 // 64MB + opsLimit = 2 + cipherText = "kBXQ2PuX6y/aje5r22H0AehRPh6sQ0ULoeAO" + cipherNonce = "v7wsI+BFZsRMIjDm3rTxPhmi/CaUdkdJ" + expectedPlainText = "plain_text" + expectedDerivedKey = "vp8d8Nee0BbIML4ab8Cp34uYnyrN77cRwTl920flyT0=" +) + +func TestDeriveArgonKey(t *testing.T) { + derivedKey, err := deriveArgonKey(password, kdfSalt, memLimit, opsLimit) + if err != nil { + t.Fatalf("Failed to derive key: %v", err) + } + + if base64.StdEncoding.EncodeToString(derivedKey) != expectedDerivedKey { + t.Fatalf("Derived key does not match expected key") + } +} + +func TestDecryptChaCha20poly1305(t *testing.T) { + derivedKey, err := deriveArgonKey(password, kdfSalt, memLimit, opsLimit) + if err != nil { + t.Fatalf("Failed to derive key: %v", err) + } + + decodedCipherText, err := base64.StdEncoding.DecodeString(cipherText) + if err != nil { + t.Fatalf("Failed to decode cipher text: %v", err) + } + + decodedCipherNonce, err := base64.StdEncoding.DecodeString(cipherNonce) + if err != nil { + t.Fatalf("Failed to decode cipher nonce: %v", err) + } + + decryptedText, err := decryptChaCha20poly1305(decodedCipherText, derivedKey, decodedCipherNonce) + if err != nil { + t.Fatalf("Failed to decrypt: %v", err) + } + if string(decryptedText) != expectedPlainText { + t.Fatalf("Decrypted text : %s does not match the expected text: %s", string(decryptedText), expectedPlainText) + } +} From 82f596097334fd51882e4d82faf094db2ebd7bba Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 6 Sep 2023 20:52:43 +0530 Subject: [PATCH 002/183] Add util method for encoding --- utils/encoding.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 utils/encoding.go diff --git a/utils/encoding.go b/utils/encoding.go new file mode 100644 index 000000000..a9a9ee6a5 --- /dev/null +++ b/utils/encoding.go @@ -0,0 +1,17 @@ +package utils + +import ( + "encoding/base64" +) + +func Base64DecodeString(s string) []byte { + b, err := base64.StdEncoding.DecodeString(s) + if err != nil { + panic(err) + } + return b +} + +func BytesToBase64(b []byte) string { + return base64.StdEncoding.EncodeToString(b) +} From edd1af66990bdab46edc14528d44ebd1573d107f Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 7 Sep 2023 08:48:00 +0530 Subject: [PATCH 003/183] Add logic to derive ente login key --- internal/crypto/crypto.go | 15 ++++++++++++++- internal/crypto/crypto_test.go | 4 ++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go index 1bd56ae14..865069fda 100644 --- a/internal/crypto/crypto.go +++ b/internal/crypto/crypto.go @@ -11,7 +11,13 @@ import ( "golang.org/x/crypto/argon2" ) -// deriveArgonKey generates a 32-bit cryptographic key using the Argon2id algorithm. +const ( + loginSubKeyLen = 32 + loginSubKeyId = 1 + loginSubKeyContext = "loginctx" +) + +// DeriveArgonKey generates a 32-bit cryptographic key using the Argon2id algorithm. // Parameters: // - password: The plaintext password to be hashed. // - salt: The salt as a base64 encoded string. @@ -68,3 +74,10 @@ func decryptChaCha20poly1305(data []byte, key []byte, nonce []byte) ([]byte, err } return decryptedData[:n], nil } + +func DeriveLoginKey(keyEncKey []byte) []byte { + mainKey := sodium.MasterKey{Bytes: keyEncKey} + subKey := mainKey.Derive(loginSubKeyLen, loginSubKeyId, loginSubKeyContext).Bytes + // return the first 16 bytes of the derived key + return subKey[:16] +} diff --git a/internal/crypto/crypto_test.go b/internal/crypto/crypto_test.go index ac59b6bbe..9f3c60bea 100644 --- a/internal/crypto/crypto_test.go +++ b/internal/crypto/crypto_test.go @@ -17,7 +17,7 @@ const ( ) func TestDeriveArgonKey(t *testing.T) { - derivedKey, err := deriveArgonKey(password, kdfSalt, memLimit, opsLimit) + derivedKey, err := DeriveArgonKey(password, kdfSalt, memLimit, opsLimit) if err != nil { t.Fatalf("Failed to derive key: %v", err) } @@ -28,7 +28,7 @@ func TestDeriveArgonKey(t *testing.T) { } func TestDecryptChaCha20poly1305(t *testing.T) { - derivedKey, err := deriveArgonKey(password, kdfSalt, memLimit, opsLimit) + derivedKey, err := DeriveArgonKey(password, kdfSalt, memLimit, opsLimit) if err != nil { t.Fatalf("Failed to derive key: %v", err) } From 2881a5effda11430a8453ce3b82e0dc7c055a77b Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 7 Sep 2023 08:53:25 +0530 Subject: [PATCH 004/183] Add basic API to perform SRP login --- internal/api/client.go | 37 +++++++++++++++++++++ internal/api/login.go | 67 ++++++++++++++++++++++++++++++++++++++ internal/api/login_type.go | 47 ++++++++++++++++++++++++++ 3 files changed, 151 insertions(+) create mode 100644 internal/api/client.go create mode 100644 internal/api/login.go create mode 100644 internal/api/login_type.go diff --git a/internal/api/client.go b/internal/api/client.go new file mode 100644 index 000000000..f24ba4ae9 --- /dev/null +++ b/internal/api/client.go @@ -0,0 +1,37 @@ +package api + +import ( + "errors" + + "github.com/go-resty/resty/v2" +) + +type Client struct { + restClient *resty.Client +} + +func NewClient() *Client { + c := resty.New() + c.EnableTrace() + c.SetError(&Error{}) + c.SetBaseURL("https://api.ente.io") + return &Client{ + restClient: c, + } +} + +// Error type for resty.Error{} +type Error struct{} + +// Implement Error() method for the error interface +func (e *Error) Error() string { + return "Error: response status code is not in the 2xx range" +} + +// OnAfterResponse Implement OnAfterResponse() method for the resty.Error interface +func (e *Error) OnAfterResponse(resp *resty.Response) error { + if resp.StatusCode() < 200 || resp.StatusCode() >= 300 { + return errors.New(e.Error()) + } + return nil +} diff --git a/internal/api/login.go b/internal/api/login.go new file mode 100644 index 000000000..df03b1f91 --- /dev/null +++ b/internal/api/login.go @@ -0,0 +1,67 @@ +package api + +import ( + "context" + "fmt" + "github.com/google/uuid" +) + +func (c *Client) GetSRPAttributes(ctx context.Context, email string) (*SRPAttributes, error) { + var res struct { + SRPAttributes SRPAttributes `json:"attributes"` + } + _, err := c.restClient.R(). + SetContext(ctx). + SetResult(&res). + SetQueryParam("email", email). + Get("/users/srp/attributes") + if err != nil { + return nil, err + } + return &res.SRPAttributes, err +} + +func (c *Client) CreateSRPSession( + ctx context.Context, + srpUserID uuid.UUID, + clientPub string, +) (*CreateSRPSessionResponse, error) { + var res CreateSRPSessionResponse + payload := map[string]interface{}{ + "srpUserID": srpUserID.String(), + "srpA": clientPub, + } + _, err := c.restClient.R(). + SetContext(ctx). + SetResult(&res). + SetBody(payload). + Post("/users/srp/create-session") + if err != nil { + return nil, err + } + return &res, nil +} + +func (c *Client) VerifySRPSession( + ctx context.Context, + srpUserID uuid.UUID, + sessionID uuid.UUID, + clientM1 string, +) (*AuthorizationResponse, error) { + var res AuthorizationResponse + payload := map[string]interface{}{ + "srpUserID": srpUserID.String(), + "sessionID": sessionID.String(), + "srpM1": clientM1, + } + r, err := c.restClient.R(). + SetContext(ctx). + SetResult(&res). + SetBody(payload). + Post("/users/srp/verify-session") + if err != nil { + return nil, err + } + fmt.Sprintf("%+v", r.RawResponse) + return &res, nil +} diff --git a/internal/api/login_type.go b/internal/api/login_type.go new file mode 100644 index 000000000..969d42cc9 --- /dev/null +++ b/internal/api/login_type.go @@ -0,0 +1,47 @@ +package api + +import ( + "github.com/google/uuid" +) + +type SRPAttributes struct { + SRPUserID uuid.UUID `json:"srpUserID" binding:"required"` + SRPSalt string `json:"srpSalt" binding:"required"` + MemLimit int `json:"memLimit" binding:"required"` + OpsLimit int `json:"opsLimit" binding:"required"` + KekSalt string `json:"kekSalt" binding:"required"` + IsEmailMFAEnabled bool `json:"isEmailMFAEnabled" binding:"required"` +} + +type CreateSRPSessionResponse struct { + SessionID uuid.UUID `json:"sessionID" binding:"required"` + SRPB string `json:"srpB" binding:"required"` +} + +// KeyAttributes stores the key related attributes for a user +type KeyAttributes struct { + KEKSalt string `json:"kekSalt" binding:"required"` + KEKHash string `json:"kekHash"` + EncryptedKey string `json:"encryptedKey" binding:"required"` + KeyDecryptionNonce string `json:"keyDecryptionNonce" binding:"required"` + PublicKey string `json:"publicKey" binding:"required"` + EncryptedSecretKey string `json:"encryptedSecretKey" binding:"required"` + SecretKeyDecryptionNonce string `json:"secretKeyDecryptionNonce" binding:"required"` + MemLimit int `json:"memLimit" binding:"required"` + OpsLimit int `json:"opsLimit" binding:"required"` + MasterKeyEncryptedWithRecoveryKey string `json:"masterKeyEncryptedWithRecoveryKey"` + MasterKeyDecryptionNonce string `json:"masterKeyDecryptionNonce"` + RecoveryKeyEncryptedWithMasterKey string `json:"recoveryKeyEncryptedWithMasterKey"` + RecoveryKeyDecryptionNonce string `json:"recoveryKeyDecryptionNonce"` +} + +type AuthorizationResponse struct { + ID int64 `json:"id"` + KeyAttributes *KeyAttributes `json:"keyAttributes,omitempty"` + EncryptedToken string `json:"encryptedToken,omitempty"` + Token string `json:"token,omitempty"` + TwoFactorSessionID string `json:"twoFactorSessionID"` + // SrpM2 is sent only if the user is logging via SRP + // SrpM2 is the SRP M2 value aka the proof that the server has the verifier + SrpM2 *string `json:"srpM2,omitempty"` +} From 2dfea90c0946103185e5aa9aee95fcd4db0ed575 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 7 Sep 2023 09:04:55 +0530 Subject: [PATCH 005/183] Add basic API to perform SRP login --- go.mod | 10 +++++++++- go.sum | 30 ++++++++++++++++++++---------- internal/crypto/crypto.go | 2 +- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 3c0c2065c..8cfc95608 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,14 @@ module cli-go go 1.20 require ( + github.com/go-resty/resty/v2 v2.7.0 + github.com/google/uuid v1.3.1 github.com/jamesruan/sodium v1.0.14 - golang.org/x/crypto v0.11.0 + golang.org/x/crypto v0.11.0 +) + +require ( + github.com/kong/go-srp v0.0.0-20191210190804-cde1efa3c083 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/sys v0.10.0 // indirect ) diff --git a/go.sum b/go.sum index f3366a91a..0f5517533 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,20 @@ -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= +github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jamesruan/sodium v1.0.14 h1:JfOHobip/lUWouxHV3PwYwu3gsLewPrDrZXO3HuBzUU= +github.com/jamesruan/sodium v1.0.14/go.mod h1:GK2+LACf7kuVQ9k7Irk0MB2B65j5rVqkz+9ylGIggZk= +github.com/kong/go-srp v0.0.0-20191210190804-cde1efa3c083 h1:Y7nibF/3Ivmk+S4Q+KzVv98lFlSdrBhYzG44d5il85E= +github.com/kong/go-srp v0.0.0-20191210190804-cde1efa3c083/go.mod h1:Zde5RRLiH8/2zEXQDHX5W0dOOTxkemzrXMhHVfxTtTA= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go index 865069fda..c11cdd046 100644 --- a/internal/crypto/crypto.go +++ b/internal/crypto/crypto.go @@ -27,7 +27,7 @@ const ( // Returns: // - A byte slice representing the derived key. // - An error object, which is nil if no error occurs. -func deriveArgonKey(password, salt string, memLimit, opsLimit int) ([]byte, error) { +func DeriveArgonKey(password, salt string, memLimit, opsLimit int) ([]byte, error) { if memLimit < 1024 || opsLimit < 1 { return nil, fmt.Errorf("invalid memory or operation limits") } From f7a90ad1ad1c55fba46a2ea4e211aee62b0488e7 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 7 Sep 2023 21:53:14 +0530 Subject: [PATCH 006/183] Add basic API to perform SRP login --- internal/api/client.go | 5 +++++ internal/api/login_type.go | 22 +++++++++------------- internal/crypto/crypto.go | 3 +++ 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/internal/api/client.go b/internal/api/client.go index f24ba4ae9..9163e66bd 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -1,6 +1,7 @@ package api import ( + "context" "errors" "github.com/go-resty/resty/v2" @@ -20,6 +21,10 @@ func NewClient() *Client { } } +func authReq(ctx context.Context, fn func(*resty.Request) (*resty.Response, error)) (*resty.Response, error) { + return fn(ctx.Value("auth").(*resty.Request)) +} + // Error type for resty.Error{} type Error struct{} diff --git a/internal/api/login_type.go b/internal/api/login_type.go index 969d42cc9..dc5003873 100644 --- a/internal/api/login_type.go +++ b/internal/api/login_type.go @@ -20,19 +20,15 @@ type CreateSRPSessionResponse struct { // KeyAttributes stores the key related attributes for a user type KeyAttributes struct { - KEKSalt string `json:"kekSalt" binding:"required"` - KEKHash string `json:"kekHash"` - EncryptedKey string `json:"encryptedKey" binding:"required"` - KeyDecryptionNonce string `json:"keyDecryptionNonce" binding:"required"` - PublicKey string `json:"publicKey" binding:"required"` - EncryptedSecretKey string `json:"encryptedSecretKey" binding:"required"` - SecretKeyDecryptionNonce string `json:"secretKeyDecryptionNonce" binding:"required"` - MemLimit int `json:"memLimit" binding:"required"` - OpsLimit int `json:"opsLimit" binding:"required"` - MasterKeyEncryptedWithRecoveryKey string `json:"masterKeyEncryptedWithRecoveryKey"` - MasterKeyDecryptionNonce string `json:"masterKeyDecryptionNonce"` - RecoveryKeyEncryptedWithMasterKey string `json:"recoveryKeyEncryptedWithMasterKey"` - RecoveryKeyDecryptionNonce string `json:"recoveryKeyDecryptionNonce"` + KEKSalt string `json:"kekSalt" binding:"required"` + KEKHash string `json:"kekHash"` + EncryptedKey string `json:"encryptedKey" binding:"required"` + KeyDecryptionNonce string `json:"keyDecryptionNonce" binding:"required"` + PublicKey string `json:"publicKey" binding:"required"` + EncryptedSecretKey string `json:"encryptedSecretKey" binding:"required"` + SecretKeyDecryptionNonce string `json:"secretKeyDecryptionNonce" binding:"required"` + MemLimit int `json:"memLimit" binding:"required"` + OpsLimit int `json:"opsLimit" binding:"required"` } type AuthorizationResponse struct { diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go index c11cdd046..84c3b2af4 100644 --- a/internal/crypto/crypto.go +++ b/internal/crypto/crypto.go @@ -75,6 +75,9 @@ func decryptChaCha20poly1305(data []byte, key []byte, nonce []byte) ([]byte, err return decryptedData[:n], nil } +// DeriveLoginKey derives a login key from the given key encryption key. +// This loginKey act as user provided password during SRP authentication. +// Parameters: keyEncKey: This is the keyEncryptionKey that is derived from the user's password. func DeriveLoginKey(keyEncKey []byte) []byte { mainKey := sodium.MasterKey{Bytes: keyEncKey} subKey := mainKey.Derive(loginSubKeyLen, loginSubKeyId, loginSubKeyContext).Bytes From 1e02b9ba368cb84142daaac36ca1781aced951f3 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 8 Sep 2023 09:32:03 +0530 Subject: [PATCH 007/183] Update gitignore --- .gitignore | 2 +- main.go | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 main.go diff --git a/.gitignore b/.gitignore index 33f94e490..de995e0ec 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,4 @@ logs/** .idea/** .vscode/** tmp/** -museum.yaml +scratch/** diff --git a/main.go b/main.go new file mode 100644 index 000000000..e51accf2d --- /dev/null +++ b/main.go @@ -0,0 +1,57 @@ +package main + +import ( + "cli-go/internal/api" + enteCrypto "cli-go/internal/crypto" + "cli-go/utils" + "context" + "fmt" + "github.com/kong/go-srp" + "os" +) + +func main() { + client := api.NewClient() + ctx := context.Background() + pass, _ := os.LookupEnv("ENTE_USER_PASSWORD") + email, _ := os.LookupEnv("ENTE_USER_EMAIL") + if pass == "" || email == "" { + fmt.Println("Please set ENTE_USER_PASSWORD and ENTE_USER_EMAIL environment variables") + return + } + srpAttr, err := client.GetSRPAttributes(ctx, email) + if err != nil { + fmt.Println(err) + } + + keyEncKey, err := enteCrypto.DeriveArgonKey(pass, srpAttr.KekSalt, srpAttr.MemLimit, srpAttr.OpsLimit) + if err != nil { + fmt.Printf("error deriving key encryption key: %v", err) + return + } + loginKey := enteCrypto.DeriveLoginKey(keyEncKey) + + srpParams := srp.GetParams(4096) + identify := []byte(srpAttr.SRPUserID.String()) + salt := utils.Base64DecodeString(srpAttr.SRPSalt) + clientSecret := srp.GenKey() + srpClient := srp.NewClient(srpParams, salt, identify, loginKey, clientSecret) + clientA := srpClient.ComputeA() + session, err := client.CreateSRPSession(ctx, srpAttr.SRPUserID, utils.BytesToBase64(clientA)) + if err != nil { + return + } + serverB := session.SRPB + srpClient.SetB(utils.Base64DecodeString(serverB)) + clientM := srpClient.ComputeM1() + authResp, err := client.VerifySRPSession(ctx, srpAttr.SRPUserID, session.SessionID, utils.BytesToBase64(clientM)) + if err != nil { + fmt.Errorf("error verifying session: %v", err) + return + } + + // generate secure random key 32 bit key + + fmt.Println("Hello World " + srpAttr.SRPUserID.String()) + fmt.Println(string(authResp.ID) + "Hello World ") +} From b6ace6dc35656dc3b4012879d45a99b190ce6ab4 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 8 Sep 2023 20:12:58 +0530 Subject: [PATCH 008/183] Add custom logger to redact headers --- internal/api/client.go | 37 +++++++++++++++++++++++++------ internal/api/log.go | 50 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 7 deletions(-) create mode 100644 internal/api/log.go diff --git a/internal/api/client.go b/internal/api/client.go index 9163e66bd..2424e2100 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -1,19 +1,46 @@ package api import ( - "context" "errors" "github.com/go-resty/resty/v2" ) +const ( + TokenHeader = "X-Auth-Token" + TokenQuery = "token" +) + +var ( + RedactedHeaders = []string{TokenHeader, " X-Request-Id"} +) + type Client struct { restClient *resty.Client + authToken *string } -func NewClient() *Client { +type Params struct { + Debug bool + Trace bool +} + +func NewClient(p Params) *Client { c := resty.New() - c.EnableTrace() + if p.Trace { + c.EnableTrace() + } + if p.Debug { + c.OnBeforeRequest(func(c *resty.Client, req *resty.Request) error { + logRequest(req) + return nil + }) + + c.OnAfterResponse(func(c *resty.Client, resp *resty.Response) error { + logResponse(resp) + return nil + }) + } c.SetError(&Error{}) c.SetBaseURL("https://api.ente.io") return &Client{ @@ -21,10 +48,6 @@ func NewClient() *Client { } } -func authReq(ctx context.Context, fn func(*resty.Request) (*resty.Response, error)) (*resty.Response, error) { - return fn(ctx.Value("auth").(*resty.Request)) -} - // Error type for resty.Error{} type Error struct{} diff --git a/internal/api/log.go b/internal/api/log.go new file mode 100644 index 000000000..87846ee0b --- /dev/null +++ b/internal/api/log.go @@ -0,0 +1,50 @@ +package api + +import ( + "fmt" + "github.com/go-resty/resty/v2" + "strings" +) + +func logRequest(req *resty.Request) { + fmt.Println("Request:") + fmt.Printf("Method: %s\n", req.Method) + fmt.Printf("URL: %s\n", req.URL) + fmt.Println("Headers:") + for k, v := range req.Header { + redacted := false + for _, rh := range RedactedHeaders { + if strings.ToLower(k) == strings.ToLower(rh) { + redacted = true + break + } + } + if redacted { + fmt.Printf("%s: %s\n", k, "REDACTED") + } else { + fmt.Printf("%s: %s\n", k, v) + } + } +} + +func logResponse(resp *resty.Response) { + fmt.Println("Response:") + fmt.Printf("Status Code: %d\n", resp.StatusCode()) + fmt.Printf("Protocol: %s\n", resp.Proto()) + fmt.Printf("Time Duration: %s\n", resp.Time()) + fmt.Println("Headers:") + for k, v := range resp.Header() { + redacted := false + for _, rh := range RedactedHeaders { + if strings.ToLower(k) == strings.ToLower(rh) { + redacted = true + break + } + } + if redacted { + fmt.Printf("%s: %s\n", k, "REDACTED") + } else { + fmt.Printf("%s: %s\n", k, v) + } + } +} From 2f0f417c8ab9d5b6701e03d402f420f1696156c1 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 8 Sep 2023 20:13:29 +0530 Subject: [PATCH 009/183] Remove redundant code --- main.go | 57 --------------------------------------------------------- 1 file changed, 57 deletions(-) delete mode 100644 main.go diff --git a/main.go b/main.go deleted file mode 100644 index e51accf2d..000000000 --- a/main.go +++ /dev/null @@ -1,57 +0,0 @@ -package main - -import ( - "cli-go/internal/api" - enteCrypto "cli-go/internal/crypto" - "cli-go/utils" - "context" - "fmt" - "github.com/kong/go-srp" - "os" -) - -func main() { - client := api.NewClient() - ctx := context.Background() - pass, _ := os.LookupEnv("ENTE_USER_PASSWORD") - email, _ := os.LookupEnv("ENTE_USER_EMAIL") - if pass == "" || email == "" { - fmt.Println("Please set ENTE_USER_PASSWORD and ENTE_USER_EMAIL environment variables") - return - } - srpAttr, err := client.GetSRPAttributes(ctx, email) - if err != nil { - fmt.Println(err) - } - - keyEncKey, err := enteCrypto.DeriveArgonKey(pass, srpAttr.KekSalt, srpAttr.MemLimit, srpAttr.OpsLimit) - if err != nil { - fmt.Printf("error deriving key encryption key: %v", err) - return - } - loginKey := enteCrypto.DeriveLoginKey(keyEncKey) - - srpParams := srp.GetParams(4096) - identify := []byte(srpAttr.SRPUserID.String()) - salt := utils.Base64DecodeString(srpAttr.SRPSalt) - clientSecret := srp.GenKey() - srpClient := srp.NewClient(srpParams, salt, identify, loginKey, clientSecret) - clientA := srpClient.ComputeA() - session, err := client.CreateSRPSession(ctx, srpAttr.SRPUserID, utils.BytesToBase64(clientA)) - if err != nil { - return - } - serverB := session.SRPB - srpClient.SetB(utils.Base64DecodeString(serverB)) - clientM := srpClient.ComputeM1() - authResp, err := client.VerifySRPSession(ctx, srpAttr.SRPUserID, session.SessionID, utils.BytesToBase64(clientM)) - if err != nil { - fmt.Errorf("error verifying session: %v", err) - return - } - - // generate secure random key 32 bit key - - fmt.Println("Hello World " + srpAttr.SRPUserID.String()) - fmt.Println(string(authResp.ID) + "Hello World ") -} From 1e2ad3067c64cafc3ddffa8603668029f7b98c3d Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Tue, 12 Sep 2023 15:26:04 +0530 Subject: [PATCH 010/183] Add models for file and collection requests --- internal/api/collection.go | 13 ++++++++++ internal/api/collection_type.go | 42 +++++++++++++++++++++++++++++++++ internal/api/file_type.go | 31 ++++++++++++++++++++++++ 3 files changed, 86 insertions(+) create mode 100644 internal/api/collection.go create mode 100644 internal/api/collection_type.go create mode 100644 internal/api/file_type.go diff --git a/internal/api/collection.go b/internal/api/collection.go new file mode 100644 index 000000000..597724981 --- /dev/null +++ b/internal/api/collection.go @@ -0,0 +1,13 @@ +package api + +import "context" + +func (c *Client) GetCollections(ctx context.Context, sinceTime int) ([]Collection, error) { + var collections []Collection + _, err := c.restClient.R(). + SetContext(ctx). + SetQueryParam("since", string(sinceTime)). + SetResult(&collections). + Get("/collections") + return collections, err +} diff --git a/internal/api/collection_type.go b/internal/api/collection_type.go new file mode 100644 index 000000000..344bffadf --- /dev/null +++ b/internal/api/collection_type.go @@ -0,0 +1,42 @@ +package api + +// Collection represents a collection +type Collection struct { + ID int64 `json:"id"` + Owner CollectionUser `json:"owner"` + EncryptedKey string `json:"encryptedKey" binding:"required"` + KeyDecryptionNonce string `json:"keyDecryptionNonce,omitempty" binding:"required"` + Name string `json:"name"` + EncryptedName string `json:"encryptedName"` + NameDecryptionNonce string `json:"nameDecryptionNonce"` + Type string `json:"type" binding:"required"` + Sharees []CollectionUser `json:"sharees"` + UpdationTime int64 `json:"updationTime"` + IsDeleted bool `json:"isDeleted,omitempty"` + MagicMetadata *MagicMetadata `json:"magicMetadata,omitempty"` + PublicMagicMetadata *MagicMetadata `json:"pubMagicMetadata,omitempty"` + SharedMagicMetadata *MagicMetadata `json:"sharedMagicMetadata,omitempty"` +} + +// CollectionUser represents the owner of a collection +type CollectionUser struct { + ID int64 `json:"id"` + Email string `json:"email"` + // Deprecated + Name string `json:"name"` + Role string `json:"role"` +} + +type MagicMetadata struct { + Version int `json:"version,omitempty" binding:"required"` + Count int `json:"count,omitempty" binding:"required"` + Data string `json:"data,omitempty" binding:"required"` + Header string `json:"header,omitempty" binding:"required"` +} + +// CollectionFileItem represents a file in an AddFilesRequest and MoveFilesRequest +type CollectionFileItem struct { + ID int64 `json:"id" binding:"required"` + EncryptedKey string `json:"encryptedKey" binding:"required"` + KeyDecryptionNonce string `json:"keyDecryptionNonce" binding:"required"` +} diff --git a/internal/api/file_type.go b/internal/api/file_type.go new file mode 100644 index 000000000..f241f8fe5 --- /dev/null +++ b/internal/api/file_type.go @@ -0,0 +1,31 @@ +package api + +// File represents an encrypted file in the system +type File struct { + ID int64 `json:"id"` + OwnerID int64 `json:"ownerID"` + CollectionID int64 `json:"collectionID"` + CollectionOwnerID *int64 `json:"collectionOwnerID"` + EncryptedKey string `json:"encryptedKey"` + KeyDecryptionNonce string `json:"keyDecryptionNonce"` + File FileAttributes `json:"file" binding:"required"` + Thumbnail FileAttributes `json:"thumbnail" binding:"required"` + Metadata FileAttributes `json:"metadata" binding:"required"` + IsDeleted bool `json:"isDeleted"` + UpdationTime int64 `json:"updationTime"` + MagicMetadata *MagicMetadata `json:"magicMetadata,omitempty"` + PubicMagicMetadata *MagicMetadata `json:"pubMagicMetadata,omitempty"` + Info *FileInfo `json:"info,omitempty"` +} + +// FileInfo has information about storage used by the file & it's metadata(future) +type FileInfo struct { + FileSize int64 `json:"fileSize,omitempty"` + ThumbnailSize int64 `json:"thumbSize,omitempty"` +} + +// FileAttributes represents a file item +type FileAttributes struct { + EncryptedData string `json:"encryptedData,omitempty"` + DecryptionHeader string `json:"decryptionHeader" binding:"required"` +} From 2c903997a3af90a1378537891169d1fcb4417384 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Tue, 12 Sep 2023 15:45:41 +0530 Subject: [PATCH 011/183] Scaffold basic cli and cmd to print version --- .gitignore | 1 + cmd/LICENSE | 0 cmd/root.go | 43 +++++++++++++++++++++++++++++++++++++++++++ cmd/version.go | 21 +++++++++++++++++++++ go.mod | 3 +++ go.sum | 10 ++++++++++ main.go | 10 ++++++++++ 7 files changed, 88 insertions(+) create mode 100644 cmd/LICENSE create mode 100644 cmd/root.go create mode 100644 cmd/version.go create mode 100644 main.go diff --git a/.gitignore b/.gitignore index de995e0ec..59d75f8de 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ logs/** .vscode/** tmp/** scratch/** +main \ No newline at end of file diff --git a/cmd/LICENSE b/cmd/LICENSE new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 000000000..e72937ee5 --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,43 @@ +package cmd + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +const AppVersion = "0.0.1" + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "ente-cli", + Short: "CLI tool for exporting your photos from ente.io", + Long: `Start by creating a config file in your home directory:`, + // Uncomment the following line if your bare application + // has an action associated with it: + Run: func(cmd *cobra.Command, args []string) { + fmt.Sprintf("Hello World") + }, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} + +func init() { + // Here you will define your flags and configuration settings. + // Cobra supports persistent flags, which, if defined here, + // will be global for your application. + + // rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.cli-go.yaml)") + + // Cobra also supports local flags, which will only run + // when this action is called directly. + rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/cmd/version.go b/cmd/version.go new file mode 100644 index 000000000..cb4d1cf47 --- /dev/null +++ b/cmd/version.go @@ -0,0 +1,21 @@ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// versionCmd represents the version command +var versionCmd = &cobra.Command{ + Use: "version", + Short: "Prints the current version", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + fmt.Printf("ente-cli version %s\n", AppVersion) + }, +} + +func init() { + rootCmd.AddCommand(versionCmd) +} diff --git a/go.mod b/go.mod index 8cfc95608..e37c5975a 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,10 @@ require ( ) require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kong/go-srp v0.0.0-20191210190804-cde1efa3c083 // indirect + github.com/spf13/cobra v1.7.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/sys v0.10.0 // indirect ) diff --git a/go.sum b/go.sum index 0f5517533..2163115f1 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,19 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jamesruan/sodium v1.0.14 h1:JfOHobip/lUWouxHV3PwYwu3gsLewPrDrZXO3HuBzUU= github.com/jamesruan/sodium v1.0.14/go.mod h1:GK2+LACf7kuVQ9k7Irk0MB2B65j5rVqkz+9ylGIggZk= github.com/kong/go-srp v0.0.0-20191210190804-cde1efa3c083 h1:Y7nibF/3Ivmk+S4Q+KzVv98lFlSdrBhYzG44d5il85E= github.com/kong/go-srp v0.0.0-20191210190804-cde1efa3c083/go.mod h1:Zde5RRLiH8/2zEXQDHX5W0dOOTxkemzrXMhHVfxTtTA= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -18,3 +26,5 @@ golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 000000000..e00cfbc3e --- /dev/null +++ b/main.go @@ -0,0 +1,10 @@ +/* +Copyright © 2023 NAME HERE +*/ +package main + +import "cli-go/cmd" + +func main() { + cmd.Execute() +} From d9803f737a5746f669dd76ed4ec51d28ffbf6fc8 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Tue, 12 Sep 2023 18:56:13 +0530 Subject: [PATCH 012/183] Add basic API to store host as config --- cmd/config.go | 61 +++++++ cmd/version.go | 2 + go.mod | 15 +- go.sum | 463 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 540 insertions(+), 1 deletion(-) create mode 100644 cmd/config.go diff --git a/cmd/config.go b/cmd/config.go new file mode 100644 index 000000000..957373ec1 --- /dev/null +++ b/cmd/config.go @@ -0,0 +1,61 @@ +package cmd + +import ( + "fmt" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +// Define the 'config' command and its subcommands +var configCmd = &cobra.Command{ + Use: "config", + Short: "Manage configuration settings", +} + +// Subcommand for 'config show' +var showCmd = &cobra.Command{ + Use: "show", + Short: "Show configuration settings", + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("host:", viper.GetString("host")) + }, +} + +// Subcommand for 'config update' +var updateCmd = &cobra.Command{ + Use: "update", + Short: "Update a configuration setting", + Run: func(cmd *cobra.Command, args []string) { + viper.Set("host", host) + err := viper.WriteConfig() + if err != nil { + fmt.Println("Error updating 'host' configuration:", err) + return + } + fmt.Println("Updating 'host' configuration:", host) + }, +} + +// Flag to specify the 'host' configuration value +var host string + +func init() { + // Set up Viper configuration + viper.SetConfigName("config") // Name of your configuration file (e.g., config.yaml) + viper.AddConfigPath(".") // Search for config file in the current directory + viper.ReadInConfig() // Read the configuration file if it exists + + // Set a default value for 'host' configuration + viper.SetDefault("host", "https://api.ente.io") + + // Add 'config' subcommands to the root command + rootCmd.AddCommand(configCmd) + + // Add flags to the 'config store' and 'config update' subcommands + updateCmd.Flags().StringVarP(&host, "host", "H", viper.GetString("host"), "Update the 'host' configuration") + // Mark 'host' flag as required for the 'update' command + updateCmd.MarkFlagRequired("host") + + // Add 'config' subcommands to the 'config' command + configCmd.AddCommand(showCmd, updateCmd) +} diff --git a/cmd/version.go b/cmd/version.go index cb4d1cf47..7568a68b4 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -13,6 +13,8 @@ var versionCmd = &cobra.Command{ Long: ``, Run: func(cmd *cobra.Command, args []string) { fmt.Printf("ente-cli version %s\n", AppVersion) + // increcement counter flag + }, } diff --git a/go.mod b/go.mod index e37c5975a..38bc439eb 100644 --- a/go.mod +++ b/go.mod @@ -10,10 +10,23 @@ require ( ) require ( + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kong/go-srp v0.0.0-20191210190804-cde1efa3c083 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/spf13/afero v1.9.5 // indirect + github.com/spf13/cast v1.5.1 // indirect github.com/spf13/cobra v1.7.0 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/viper v1.16.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.10.0 // indirect + golang.org/x/sys v0.12.0 // indirect + golang.org/x/text v0.13.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 2163115f1..fcb9020cf 100644 --- a/go.sum +++ b/go.sum @@ -1,30 +1,493 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jamesruan/sodium v1.0.14 h1:JfOHobip/lUWouxHV3PwYwu3gsLewPrDrZXO3HuBzUU= github.com/jamesruan/sodium v1.0.14/go.mod h1:GK2+LACf7kuVQ9k7Irk0MB2B65j5rVqkz+9ylGIggZk= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kong/go-srp v0.0.0-20191210190804-cde1efa3c083 h1:Y7nibF/3Ivmk+S4Q+KzVv98lFlSdrBhYzG44d5il85E= github.com/kong/go-srp v0.0.0-20191210190804-cde1efa3c083/go.mod h1:Zde5RRLiH8/2zEXQDHX5W0dOOTxkemzrXMhHVfxTtTA= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= +github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= +github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= +github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= +github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= +github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From 4c06ed01121fde05ef674c352b366136110ec685 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 13 Sep 2023 08:20:15 +0530 Subject: [PATCH 013/183] Clean up --- cmd/config.go | 4 ---- cmd/root.go | 4 ++++ config.yaml | 1 + pkg/account.go | 1 + 4 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 config.yaml create mode 100644 pkg/account.go diff --git a/cmd/config.go b/cmd/config.go index 957373ec1..896edec53 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -41,10 +41,6 @@ var host string func init() { // Set up Viper configuration - viper.SetConfigName("config") // Name of your configuration file (e.g., config.yaml) - viper.AddConfigPath(".") // Search for config file in the current directory - viper.ReadInConfig() // Read the configuration file if it exists - // Set a default value for 'host' configuration viper.SetDefault("host", "https://api.ente.io") diff --git a/cmd/root.go b/cmd/root.go index e72937ee5..343e5da39 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "github.com/spf13/viper" "os" "github.com/spf13/cobra" @@ -40,4 +41,7 @@ func init() { // Cobra also supports local flags, which will only run // when this action is called directly. rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") + viper.SetConfigName("config") // Name of your configuration file (e.g., config.yaml) + viper.AddConfigPath(".") // Search for config file in the current directory + viper.ReadInConfig() // Read the configuration file if it exists } diff --git a/config.yaml b/config.yaml new file mode 100644 index 000000000..fea893ae2 --- /dev/null +++ b/config.yaml @@ -0,0 +1 @@ +host: https://api.ente.io diff --git a/pkg/account.go b/pkg/account.go new file mode 100644 index 000000000..c1caffeb1 --- /dev/null +++ b/pkg/account.go @@ -0,0 +1 @@ +package pkg From a98aa9f4ea0524f32830d23289eb0b81a6a8bf5b Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 13 Sep 2023 13:39:39 +0530 Subject: [PATCH 014/183] Add bbolt for storage --- go.mod | 1 + go.sum | 2 ++ pkg/bolt_db.go | 16 ++++++++++++++++ 3 files changed, 19 insertions(+) create mode 100644 pkg/bolt_db.go diff --git a/go.mod b/go.mod index 38bc439eb..bc75de160 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.16.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect + go.etcd.io/bbolt v1.3.7 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/sys v0.12.0 // indirect golang.org/x/text v0.13.0 // indirect diff --git a/go.sum b/go.sum index fcb9020cf..631e3ac55 100644 --- a/go.sum +++ b/go.sum @@ -177,6 +177,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= +go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/pkg/bolt_db.go b/pkg/bolt_db.go new file mode 100644 index 000000000..533ac71b1 --- /dev/null +++ b/pkg/bolt_db.go @@ -0,0 +1,16 @@ +package pkg + +import ( + "log" + "time" + + bolt "go.etcd.io/bbolt" +) + +func GetDB(path string) (*bolt.DB, error) { + db, err := bolt.Open(path, 0600, &bolt.Options{Timeout: 1 * time.Second}) + if err != nil { + log.Fatal(err) + } + return db, err +} From 181056d8c5c5f24cd58dfa85242b88d3492a8424 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 13 Sep 2023 14:19:56 +0530 Subject: [PATCH 015/183] APIClient: Pass host as parameter --- internal/api/client.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/internal/api/client.go b/internal/api/client.go index 2424e2100..6d5c952ba 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -7,8 +7,9 @@ import ( ) const ( - TokenHeader = "X-Auth-Token" - TokenQuery = "token" + EnteAPIEndpoint = "https://api.ente.io" + TokenHeader = "X-Auth-Token" + TokenQuery = "token" ) var ( @@ -23,6 +24,7 @@ type Client struct { type Params struct { Debug bool Trace bool + Host string } func NewClient(p Params) *Client { @@ -42,7 +44,11 @@ func NewClient(p Params) *Client { }) } c.SetError(&Error{}) - c.SetBaseURL("https://api.ente.io") + if p.Host != "" { + c.SetBaseURL(p.Host) + } else { + c.SetBaseURL(EnteAPIEndpoint) + } return &Client{ restClient: c, } From 57ad458662a282d58df585c4234fcf1d101dc53e Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 13 Sep 2023 14:21:05 +0530 Subject: [PATCH 016/183] Add pkg ctrl --- .gitignore | 4 +++- cmd/root.go | 6 +++++- internal/api/login.go | 4 +--- main.go | 38 ++++++++++++++++++++++++++++++++++++-- pkg/account.go | 29 +++++++++++++++++++++++++++++ pkg/controller.go | 11 +++++++++++ 6 files changed, 85 insertions(+), 7 deletions(-) create mode 100644 pkg/controller.go diff --git a/.gitignore b/.gitignore index 59d75f8de..d430455b6 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,6 @@ logs/** .vscode/** tmp/** scratch/** -main \ No newline at end of file +main +config.yaml +ente-cli.db \ No newline at end of file diff --git a/cmd/root.go b/cmd/root.go index 343e5da39..b31c2fa69 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,6 +1,7 @@ package cmd import ( + "cli-go/pkg" "fmt" "github.com/spf13/viper" "os" @@ -10,6 +11,8 @@ import ( const AppVersion = "0.0.1" +var ctrl *pkg.ClICtrl + // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: "ente-cli", @@ -24,7 +27,8 @@ var rootCmd = &cobra.Command{ // Execute adds all child commands to the root command and sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute() { +func Execute(controller *pkg.ClICtrl) { + ctrl = controller err := rootCmd.Execute() if err != nil { os.Exit(1) diff --git a/internal/api/login.go b/internal/api/login.go index df03b1f91..0c5ad91b4 100644 --- a/internal/api/login.go +++ b/internal/api/login.go @@ -2,7 +2,6 @@ package api import ( "context" - "fmt" "github.com/google/uuid" ) @@ -54,7 +53,7 @@ func (c *Client) VerifySRPSession( "sessionID": sessionID.String(), "srpM1": clientM1, } - r, err := c.restClient.R(). + _, err := c.restClient.R(). SetContext(ctx). SetResult(&res). SetBody(payload). @@ -62,6 +61,5 @@ func (c *Client) VerifySRPSession( if err != nil { return nil, err } - fmt.Sprintf("%+v", r.RawResponse) return &res, nil } diff --git a/main.go b/main.go index e00cfbc3e..79c2e6fbc 100644 --- a/main.go +++ b/main.go @@ -3,8 +3,42 @@ Copyright © 2023 NAME HERE */ package main -import "cli-go/cmd" +import ( + "cli-go/cmd" + "cli-go/internal/api" + "cli-go/pkg" + "fmt" + bolt "go.etcd.io/bbolt" + "log" +) + +var db *bolt.DB +var client = api.NewClient(api.Params{ + Debug: true, +}) func main() { - cmd.Execute() + db, err := pkg.GetDB("ente-cli.db") + if err != nil { + panic(err) + } + db.Update(func(tx *bolt.Tx) error { + log.Println("creating #AccBucket") + _, err := tx.CreateBucketIfNotExists([]byte(pkg.AccBucket)) + if err != nil { + return fmt.Errorf("create bucket: %s", err) + } + return nil + }) + ctrl := pkg.ClICtrl{ + Client: client, + DB: db, + } + defer func() { + log.Println("closing db") + if err := db.Close(); err != nil { + panic(err) + } + }() + cmd.Execute(&ctrl) } diff --git a/pkg/account.go b/pkg/account.go index c1caffeb1..c02106df5 100644 --- a/pkg/account.go +++ b/pkg/account.go @@ -1 +1,30 @@ package pkg + +import ( + "context" + "encoding/json" + bolt "go.etcd.io/bbolt" +) + +const AccBucket = "accounts" + +type AccountInfo struct { + Email string `json:"email" binding:"required"` + UserID int64 `json:"userID" binding:"required"` +} + +func (c *ClICtrl) AddAccount(cxt context.Context, email string, userID int64) error { + return c.DB.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(AccBucket)) + info := AccountInfo{ + Email: email, + UserID: userID, + } + value, err := json.Marshal(info) + if err != nil { + return err + } + err = b.Put([]byte(email), value) + return err + }) +} diff --git a/pkg/controller.go b/pkg/controller.go new file mode 100644 index 000000000..c1eb5eb97 --- /dev/null +++ b/pkg/controller.go @@ -0,0 +1,11 @@ +package pkg + +import ( + "cli-go/internal/api" + bolt "go.etcd.io/bbolt" +) + +type ClICtrl struct { + Client *api.Client + DB *bolt.DB +} From f34be1f3c01544542e4c417636b378fb68ae1221 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 13 Sep 2023 18:09:54 +0530 Subject: [PATCH 017/183] Add colors to http log --- go.mod | 6 ++++++ go.sum | 9 +++++++++ internal/api/log.go | 38 +++++++++++++++++++++++--------------- 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index bc75de160..83a17768e 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,12 @@ require ( ) require ( + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect +) + +require ( + github.com/fatih/color v1.15.0 github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect diff --git a/go.sum b/go.sum index 631e3ac55..252cd66a5 100644 --- a/go.sum +++ b/go.sum @@ -55,6 +55,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -139,6 +141,11 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= @@ -322,6 +329,8 @@ golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/internal/api/log.go b/internal/api/log.go index 87846ee0b..ba97c0d86 100644 --- a/internal/api/log.go +++ b/internal/api/log.go @@ -2,49 +2,57 @@ package api import ( "fmt" - "github.com/go-resty/resty/v2" "strings" + + "github.com/fatih/color" + "github.com/go-resty/resty/v2" ) func logRequest(req *resty.Request) { - fmt.Println("Request:") - fmt.Printf("Method: %s\n", req.Method) - fmt.Printf("URL: %s\n", req.URL) - fmt.Println("Headers:") + fmt.Println(color.GreenString("Request:")) + fmt.Printf("%s %s\n", color.CyanString(req.Method), color.YellowString(req.URL)) + fmt.Println(color.GreenString("Headers:")) for k, v := range req.Header { redacted := false for _, rh := range RedactedHeaders { - if strings.ToLower(k) == strings.ToLower(rh) { + if strings.EqualFold(strings.ToLower(k), strings.ToLower(rh)) { redacted = true break } } if redacted { - fmt.Printf("%s: %s\n", k, "REDACTED") + fmt.Printf("%s: %s\n", color.CyanString(k), color.RedString("REDACTED")) } else { - fmt.Printf("%s: %s\n", k, v) + if len(v) == 1 { + fmt.Printf("%s: %s\n", color.CyanString(k), color.YellowString(v[0])) + } else { + fmt.Printf("%s: %s\n", color.CyanString(k), color.YellowString(strings.Join(v, ","))) + } } } } func logResponse(resp *resty.Response) { - fmt.Println("Response:") - fmt.Printf("Status Code: %d\n", resp.StatusCode()) - fmt.Printf("Protocol: %s\n", resp.Proto()) + fmt.Println(color.GreenString("Response:")) + if resp.StatusCode() < 200 || resp.StatusCode() >= 300 { + fmt.Printf("%s %s\n", color.CyanString(resp.Proto()), color.RedString(resp.Status())) + } else { + fmt.Printf("%s %s\n", color.CyanString(resp.Proto()), color.YellowString(resp.Status())) + } fmt.Printf("Time Duration: %s\n", resp.Time()) - fmt.Println("Headers:") + fmt.Println(color.GreenString("Headers:")) for k, v := range resp.Header() { redacted := false for _, rh := range RedactedHeaders { - if strings.ToLower(k) == strings.ToLower(rh) { + if strings.EqualFold(strings.ToLower(k), strings.ToLower(rh)) { redacted = true break } } if redacted { - fmt.Printf("%s: %s\n", k, "REDACTED") + fmt.Printf("%s: %s\n", color.CyanString(k), color.RedString("REDACTED")) } else { - fmt.Printf("%s: %s\n", k, v) + fmt.Printf("%s: %s\n", color.CyanString(k), color.YellowString(strings.Join(v, ","))) } } } From 06040cbfc4940190407c66a0f6815bb9bfb72d68 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 13 Sep 2023 18:28:15 +0530 Subject: [PATCH 018/183] Add colors to http log --- go.mod | 3 ++- go.sum | 3 +++ pkg/promt.go | 31 +++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 pkg/promt.go diff --git a/go.mod b/go.mod index 83a17768e..34c002048 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/go-resty/resty/v2 v2.7.0 github.com/google/uuid v1.3.1 github.com/jamesruan/sodium v1.0.14 - golang.org/x/crypto v0.11.0 + golang.org/x/crypto v0.13.0 ) require ( @@ -33,6 +33,7 @@ require ( go.etcd.io/bbolt v1.3.7 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/sys v0.12.0 // indirect + golang.org/x/term v0.12.0 // indirect golang.org/x/text v0.13.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 252cd66a5..4a5b4432c 100644 --- a/go.sum +++ b/go.sum @@ -201,6 +201,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -323,6 +325,7 @@ golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/pkg/promt.go b/pkg/promt.go new file mode 100644 index 000000000..d1ebfbd00 --- /dev/null +++ b/pkg/promt.go @@ -0,0 +1,31 @@ +package pkg + +import ( + "errors" + "fmt" + "os" + + "golang.org/x/term" +) + +func GetSensitiveField(label string) (string, error) { + fmt.Printf("%s: ", label) + input, err := term.ReadPassword(int(os.Stdin.Fd())) + if err != nil { + return "", err + } + return string(input), nil +} + +func GetUserInput(label string) (string, error) { + fmt.Printf("%s: ", label) + var input string + _, err := fmt.Scanln(&input) + if err != nil { + return "", err + } + if input == "" { + return "", errors.New("input cannot be empty") + } + return input, nil +} From 1bc363ac8d203c6ce6f2c0ddcf4f8d321244a9a4 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 14 Sep 2023 09:50:32 +0530 Subject: [PATCH 019/183] Add basic commands for account --- cmd/account.go | 40 ++++++++++++++++ cmd/root.go | 3 +- internal/api/client.go | 19 -------- internal/api/enums.go | 9 ++++ internal/api/login.go | 81 ++++++++++++++++++++++++++++++-- internal/api/login_type.go | 15 ++++++ main.go | 15 +++--- pkg/account.go | 95 +++++++++++++++++++++++++++++++++----- pkg/promt.go | 22 +++++++++ pkg/sign_in.go | 50 ++++++++++++++++++++ 10 files changed, 304 insertions(+), 45 deletions(-) create mode 100644 cmd/account.go create mode 100644 internal/api/enums.go create mode 100644 pkg/sign_in.go diff --git a/cmd/account.go b/cmd/account.go new file mode 100644 index 000000000..454d966a6 --- /dev/null +++ b/cmd/account.go @@ -0,0 +1,40 @@ +package cmd + +import ( + "context" + + "github.com/spf13/cobra" +) + +// Define the 'account' command and its subcommands +var accountCmd = &cobra.Command{ + Use: "account", + Short: "Manage account settings", +} + +// Subcommand for 'account list' +var listAccCmd = &cobra.Command{ + Use: "list", + Short: "list configured accounts", + RunE: func(cmd *cobra.Command, args []string) error { + return ctrl.ListAccounts(context.Background()) + }, +} + +// Subcommand for 'account add' +var addAccCmd = &cobra.Command{ + Use: "add", + Short: "Add a new account", + Run: func(cmd *cobra.Command, args []string) { + ctrl.AddAccount(context.Background()) + }, +} + +func init() { + + // Add 'config' subcommands to the root command + rootCmd.AddCommand(accountCmd) + + // Add 'config' subcommands to the 'config' command + accountCmd.AddCommand(listAccCmd, addAccCmd) +} diff --git a/cmd/root.go b/cmd/root.go index b31c2fa69..0b30ea92f 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -3,9 +3,10 @@ package cmd import ( "cli-go/pkg" "fmt" - "github.com/spf13/viper" "os" + "github.com/spf13/viper" + "github.com/spf13/cobra" ) diff --git a/internal/api/client.go b/internal/api/client.go index 6d5c952ba..5c73866c3 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -1,8 +1,6 @@ package api import ( - "errors" - "github.com/go-resty/resty/v2" ) @@ -43,7 +41,6 @@ func NewClient(p Params) *Client { return nil }) } - c.SetError(&Error{}) if p.Host != "" { c.SetBaseURL(p.Host) } else { @@ -53,19 +50,3 @@ func NewClient(p Params) *Client { restClient: c, } } - -// Error type for resty.Error{} -type Error struct{} - -// Implement Error() method for the error interface -func (e *Error) Error() string { - return "Error: response status code is not in the 2xx range" -} - -// OnAfterResponse Implement OnAfterResponse() method for the resty.Error interface -func (e *Error) OnAfterResponse(resp *resty.Response) error { - if resp.StatusCode() < 200 || resp.StatusCode() >= 300 { - return errors.New(e.Error()) - } - return nil -} diff --git a/internal/api/enums.go b/internal/api/enums.go new file mode 100644 index 000000000..42cddc287 --- /dev/null +++ b/internal/api/enums.go @@ -0,0 +1,9 @@ +package api + +type App string + +const ( + AppPhotos App = "photos" + AppAuth App = "auth" + AppLocker App = "locker" +) diff --git a/internal/api/login.go b/internal/api/login.go index 0c5ad91b4..8067384fc 100644 --- a/internal/api/login.go +++ b/internal/api/login.go @@ -2,14 +2,15 @@ package api import ( "context" + "github.com/google/uuid" ) func (c *Client) GetSRPAttributes(ctx context.Context, email string) (*SRPAttributes, error) { var res struct { - SRPAttributes SRPAttributes `json:"attributes"` + SRPAttributes *SRPAttributes `json:"attributes"` } - _, err := c.restClient.R(). + r, err := c.restClient.R(). SetContext(ctx). SetResult(&res). SetQueryParam("email", email). @@ -17,7 +18,13 @@ func (c *Client) GetSRPAttributes(ctx context.Context, email string) (*SRPAttrib if err != nil { return nil, err } - return &res.SRPAttributes, err + if r.IsError() { + return nil, &ApiError{ + StatusCode: r.StatusCode(), + Message: r.String(), + } + } + return res.SRPAttributes, err } func (c *Client) CreateSRPSession( @@ -30,7 +37,7 @@ func (c *Client) CreateSRPSession( "srpUserID": srpUserID.String(), "srpA": clientPub, } - _, err := c.restClient.R(). + r, err := c.restClient.R(). SetContext(ctx). SetResult(&res). SetBody(payload). @@ -38,6 +45,12 @@ func (c *Client) CreateSRPSession( if err != nil { return nil, err } + if r.IsError() { + return nil, &ApiError{ + StatusCode: r.StatusCode(), + Message: r.String(), + } + } return &res, nil } @@ -53,7 +66,7 @@ func (c *Client) VerifySRPSession( "sessionID": sessionID.String(), "srpM1": clientM1, } - _, err := c.restClient.R(). + r, err := c.restClient.R(). SetContext(ctx). SetResult(&res). SetBody(payload). @@ -61,5 +74,63 @@ func (c *Client) VerifySRPSession( if err != nil { return nil, err } + if r.IsError() { + return nil, &ApiError{ + StatusCode: r.StatusCode(), + Message: r.String(), + } + } + return &res, nil +} + +func (c *Client) SendEmailOTP( + ctx context.Context, + email string, +) error { + var res AuthorizationResponse + payload := map[string]interface{}{ + "email": email, + } + r, err := c.restClient.R(). + SetContext(ctx). + SetResult(&res). + SetBody(payload). + Post("/users/ott") + if err != nil { + return err + } + if r.IsError() { + return &ApiError{ + StatusCode: r.StatusCode(), + Message: r.String(), + } + } + return nil +} + +func (c *Client) VerifyEmail( + ctx context.Context, + email string, + otp string, +) (*AuthorizationResponse, error) { + var res AuthorizationResponse + payload := map[string]interface{}{ + "email": email, + "ott": otp, + } + r, err := c.restClient.R(). + SetContext(ctx). + SetResult(&res). + SetBody(payload). + Post("/users/verify-email") + if err != nil { + return nil, err + } + if r.IsError() { + return nil, &ApiError{ + StatusCode: r.StatusCode(), + Message: r.String(), + } + } return &res, nil } diff --git a/internal/api/login_type.go b/internal/api/login_type.go index dc5003873..9c60d4eb1 100644 --- a/internal/api/login_type.go +++ b/internal/api/login_type.go @@ -1,6 +1,8 @@ package api import ( + "fmt" + "github.com/google/uuid" ) @@ -13,6 +15,15 @@ type SRPAttributes struct { IsEmailMFAEnabled bool `json:"isEmailMFAEnabled" binding:"required"` } +type ApiError struct { + Message string + StatusCode int +} + +func (e *ApiError) Error() string { + return fmt.Sprintf("status %d with err: %s", e.StatusCode, e.Message) +} + type CreateSRPSessionResponse struct { SessionID uuid.UUID `json:"sessionID" binding:"required"` SRPB string `json:"srpB" binding:"required"` @@ -41,3 +52,7 @@ type AuthorizationResponse struct { // SrpM2 is the SRP M2 value aka the proof that the server has the verifier SrpM2 *string `json:"srpM2,omitempty"` } + +func (a *AuthorizationResponse) IsMFARequired() bool { + return a.TwoFactorSessionID != "" +} diff --git a/main.go b/main.go index 79c2e6fbc..418167072 100644 --- a/main.go +++ b/main.go @@ -8,14 +8,9 @@ import ( "cli-go/internal/api" "cli-go/pkg" "fmt" - bolt "go.etcd.io/bbolt" - "log" -) -var db *bolt.DB -var client = api.NewClient(api.Params{ - Debug: true, -}) + bolt "go.etcd.io/bbolt" +) func main() { db, err := pkg.GetDB("ente-cli.db") @@ -23,19 +18,21 @@ func main() { panic(err) } db.Update(func(tx *bolt.Tx) error { - log.Println("creating #AccBucket") _, err := tx.CreateBucketIfNotExists([]byte(pkg.AccBucket)) if err != nil { return fmt.Errorf("create bucket: %s", err) } return nil }) + var client = api.NewClient(api.Params{ + Debug: false, + Host: "http://localhost:8080", + }) ctrl := pkg.ClICtrl{ Client: client, DB: db, } defer func() { - log.Println("closing db") if err := db.Close(); err != nil { panic(err) } diff --git a/pkg/account.go b/pkg/account.go index c02106df5..b85987124 100644 --- a/pkg/account.go +++ b/pkg/account.go @@ -1,30 +1,103 @@ package pkg import ( + "cli-go/internal/api" "context" "encoding/json" + "fmt" + "log" + bolt "go.etcd.io/bbolt" ) const AccBucket = "accounts" type AccountInfo struct { - Email string `json:"email" binding:"required"` - UserID int64 `json:"userID" binding:"required"` + Email string `json:"email" binding:"required"` + UserID int64 `json:"userID" binding:"required"` + App api.App `json:"app" binding:"required"` } -func (c *ClICtrl) AddAccount(cxt context.Context, email string, userID int64) error { - return c.DB.Update(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte(AccBucket)) - info := AccountInfo{ - Email: email, - UserID: userID, +func (c *ClICtrl) AddAccount(cxt context.Context) { + var flowErr error + defer func() { + if r := recover(); r != nil { + log.Println("recovered from panic", r) } - value, err := json.Marshal(info) + if flowErr != nil { + log.Fatal(flowErr) + } + }() + + email, flowErr := GetUserInput("Enter email address") + if flowErr != nil { + return + } + var keyEncKey []byte + var verifyEmail bool + var authResponse *api.AuthorizationResponse + srpAttr, flowErr := c.Client.GetSRPAttributes(cxt, email) + if flowErr != nil { + return + } + if verifyEmail || srpAttr.IsEmailMFAEnabled { + flowErr = c.Client.SendEmailOTP(cxt, email) + if flowErr != nil { + return + } + ott, otpErr := GetCode( + fmt.Sprintf("Enter OTP sent to email %s (or 'c' to cancel)", email), + 6) + if otpErr != nil { + flowErr = otpErr + return + } + authResponse, flowErr = c.Client.VerifyEmail(cxt, email, ott) + } else { + authResponse, keyEncKey, flowErr = c.signInViaPassword(cxt, email, srpAttr) + } + if flowErr != nil { + return + } + if authResponse == nil { + return + } + if authResponse.IsMFARequired() { + + } + if keyEncKey == nil { + pass, flowErr := GetSensitiveField("Enter password") + if flowErr != nil { + return + } else if pass == "" { + log.Printf("do work") + } + } +} + +func (c *ClICtrl) ListAccounts(cxt context.Context) error { + err := c.DB.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(AccBucket)) + log.Printf("total accounts: %d", b.Stats().KeyN) + err := b.ForEach(func(k, v []byte) error { + var info AccountInfo + err := json.Unmarshal(v, &info) + if err != nil { + return err + } + log.Println(info) + return nil + }) if err != nil { + log.Fatal("error listing accounts", err) return err } - err = b.Put([]byte(email), value) - return err + return nil }) + if err != nil { + log.Fatal("error listing accounts", err) + return err + } + return err + } diff --git a/pkg/promt.go b/pkg/promt.go index d1ebfbd00..32936f469 100644 --- a/pkg/promt.go +++ b/pkg/promt.go @@ -3,6 +3,7 @@ package pkg import ( "errors" "fmt" + "log" "os" "golang.org/x/term" @@ -29,3 +30,24 @@ func GetUserInput(label string) (string, error) { } return input, nil } + +func GetCode(promptText string, length int) (string, error) { + for { + ott, err := GetUserInput(promptText) + if err != nil { + return "", err + } + if ott == "" { + log.Fatal("no OTP entered") + return "", errors.New("no OTP entered") + } + if ott == "c" { + return "", errors.New("OTP entry cancelled") + } + if len(ott) != length { + fmt.Printf("OTP must be %d digits", length) + continue + } + return ott, nil + } +} diff --git a/pkg/sign_in.go b/pkg/sign_in.go new file mode 100644 index 000000000..d66c6fbad --- /dev/null +++ b/pkg/sign_in.go @@ -0,0 +1,50 @@ +package pkg + +import ( + "cli-go/internal/api" + enteCrypto "cli-go/internal/crypto" + "cli-go/utils" + "context" + "fmt" + "log" + + "github.com/kong/go-srp" +) + +func (c *ClICtrl) signInViaPassword(ctx context.Context, email string, srpAttr *api.SRPAttributes) (*api.AuthorizationResponse, []byte, error) { + for { + // CLI prompt for password + password, flowErr := GetSensitiveField("Enter password") + if flowErr != nil { + return nil, nil, flowErr + } + fmt.Println("\nPlease wait authenticating...") + keyEncKey, err := enteCrypto.DeriveArgonKey(password, srpAttr.KekSalt, srpAttr.MemLimit, srpAttr.OpsLimit) + if err != nil { + fmt.Printf("error deriving key encryption key: %v", err) + return nil, nil, err + } + loginKey := enteCrypto.DeriveLoginKey(keyEncKey) + + srpParams := srp.GetParams(4096) + identify := []byte(srpAttr.SRPUserID.String()) + salt := utils.Base64DecodeString(srpAttr.SRPSalt) + clientSecret := srp.GenKey() + srpClient := srp.NewClient(srpParams, salt, identify, loginKey, clientSecret) + clientA := srpClient.ComputeA() + session, err := c.Client.CreateSRPSession(ctx, srpAttr.SRPUserID, utils.BytesToBase64(clientA)) + if err != nil { + return nil, nil, err + } + serverB := session.SRPB + srpClient.SetB(utils.Base64DecodeString(serverB)) + clientM := srpClient.ComputeM1() + authResp, err := c.Client.VerifySRPSession(ctx, srpAttr.SRPUserID, session.SessionID, utils.BytesToBase64(clientM)) + if err != nil { + log.Printf("failed to verify %v", err) + continue + } + return authResp, keyEncKey, nil + } + +} From eaf73c2f3fe2f41ec5dbee77700f8b9a5366b24a Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 14 Sep 2023 12:24:09 +0530 Subject: [PATCH 020/183] Add script to create binary --- .gitignore | 3 ++- main.go | 3 --- release.sh | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 4 deletions(-) create mode 100755 release.sh diff --git a/.gitignore b/.gitignore index d430455b6..c90d2b2b9 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ tmp/** scratch/** main config.yaml -ente-cli.db \ No newline at end of file +ente-cli.db +bin/** \ No newline at end of file diff --git a/main.go b/main.go index 418167072..27d9428ed 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,3 @@ -/* -Copyright © 2023 NAME HERE -*/ package main import ( diff --git a/release.sh b/release.sh new file mode 100755 index 000000000..883aa4d87 --- /dev/null +++ b/release.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Create a "bin" directory if it doesn't exist +mkdir -p bin + +# List of target operating systems +OS_TARGETS=("windows" "linux" "darwin") + +# Loop through each OS target +for OS in "${OS_TARGETS[@]}" +do + # Set the GOOS environment variable for the current target OS + export GOOS="$OS" + + # Set the output binary name to "ente-cli" for the current OS + BINARY_NAME="ente-cli" + + # Add .exe extension for Windows + if [ "$OS" == "windows" ]; then + BINARY_NAME="ente-cli.exe" + fi + + # Add .exe extension for Windows + if [ "$OS" == "darwin" ]; then + BINARY_NAME="ente-cli-mac" + fi + + + + # Build the binary and place it in the "bin" directory + go build -o "bin/$BINARY_NAME" main.go + + # Print a message indicating the build is complete for the current OS + echo "Built for $OS as bin/$BINARY_NAME" +done + +# Clean up any environment variables +unset GOOS + +# Print a message indicating the build process is complete +echo "Build process completed for all platforms. Binaries are in the 'bin' directory." From 018833c543d0ce8981e739d9b10f3b8e0f8ea9fb Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 14 Sep 2023 13:03:20 +0530 Subject: [PATCH 021/183] Add readme --- README.md | 20 ++++++++++++++++++++ config.yaml | 1 - release.sh | 4 ++-- 3 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 000000000..7af30add8 --- /dev/null +++ b/README.md @@ -0,0 +1,20 @@ +# cli tool for exporting ente photos + +## Testing + +Run the release script to build the binary and run it. + +```shell + ./release.sh +``` + +or you can run the following command + +```shell + go build -o "bin/ente-cli" main.go +``` + +```shell +./bin/ente-cli --help +``` + diff --git a/config.yaml b/config.yaml index fea893ae2..e69de29bb 100644 --- a/config.yaml +++ b/config.yaml @@ -1 +0,0 @@ -host: https://api.ente.io diff --git a/release.sh b/release.sh index 883aa4d87..f3886da41 100755 --- a/release.sh +++ b/release.sh @@ -24,8 +24,8 @@ do if [ "$OS" == "darwin" ]; then BINARY_NAME="ente-cli-mac" fi - - + # make bin directory if it doesn't exist + mkdir -p bin # Build the binary and place it in the "bin" directory go build -o "bin/$BINARY_NAME" main.go From d442dff14a4e507256d23a2be1fc46c3ef87b6e1 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 15 Sep 2023 16:17:38 +0530 Subject: [PATCH 022/183] Add support for validating 2FA --- cmd/version.go | 2 -- internal/api/login.go | 27 +++++++++++++++++++++++++++ pkg/account.go | 5 +---- pkg/sign_in.go | 20 +++++++++++++++++++- 4 files changed, 47 insertions(+), 7 deletions(-) diff --git a/cmd/version.go b/cmd/version.go index 7568a68b4..cb4d1cf47 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -13,8 +13,6 @@ var versionCmd = &cobra.Command{ Long: ``, Run: func(cmd *cobra.Command, args []string) { fmt.Printf("ente-cli version %s\n", AppVersion) - // increcement counter flag - }, } diff --git a/internal/api/login.go b/internal/api/login.go index 8067384fc..cb6180cc6 100644 --- a/internal/api/login.go +++ b/internal/api/login.go @@ -134,3 +134,30 @@ func (c *Client) VerifyEmail( } return &res, nil } + +func (c *Client) VerifyTotp( + ctx context.Context, + sessionID string, + otp string, +) (*AuthorizationResponse, error) { + var res AuthorizationResponse + payload := map[string]interface{}{ + "sessionID": sessionID, + "code": otp, + } + r, err := c.restClient.R(). + SetContext(ctx). + SetResult(&res). + SetBody(payload). + Post("/users/two-factor/verify") + if err != nil { + return nil, err + } + if r.IsError() { + return nil, &ApiError{ + StatusCode: r.StatusCode(), + Message: r.String(), + } + } + return &res, nil +} diff --git a/pkg/account.go b/pkg/account.go index b85987124..8c8060910 100644 --- a/pkg/account.go +++ b/pkg/account.go @@ -59,11 +59,8 @@ func (c *ClICtrl) AddAccount(cxt context.Context) { if flowErr != nil { return } - if authResponse == nil { - return - } if authResponse.IsMFARequired() { - + authResponse, flowErr = c.validateTOTP(cxt, authResponse) } if keyEncKey == nil { pass, flowErr := GetSensitiveField("Enter password") diff --git a/pkg/sign_in.go b/pkg/sign_in.go index d66c6fbad..7507e8ceb 100644 --- a/pkg/sign_in.go +++ b/pkg/sign_in.go @@ -46,5 +46,23 @@ func (c *ClICtrl) signInViaPassword(ctx context.Context, email string, srpAttr * } return authResp, keyEncKey, nil } - +} + +func (c *ClICtrl) validateTOTP(ctx context.Context, authResp *api.AuthorizationResponse) (*api.AuthorizationResponse, error) { + if !authResp.IsMFARequired() { + return authResp, nil + } + for { + // CLI prompt for TOTP + totp, flowErr := GetCode("Enter TOTP", 6) + if flowErr != nil { + return nil, flowErr + } + totpResp, err := c.Client.VerifyTotp(ctx, authResp.TwoFactorSessionID, totp) + if err != nil { + log.Printf("failed to verify %v", err) + continue + } + return totpResp, nil + } } From 3a278a54467b03eee6c68442fee0a0769d9581ce Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 15 Sep 2023 16:25:29 +0530 Subject: [PATCH 023/183] Refactor --- main.go | 25 +++++++++---------------- pkg/controller.go | 11 +++++++++++ 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/main.go b/main.go index 27d9428ed..2e72e97f0 100644 --- a/main.go +++ b/main.go @@ -4,9 +4,6 @@ import ( "cli-go/cmd" "cli-go/internal/api" "cli-go/pkg" - "fmt" - - bolt "go.etcd.io/bbolt" ) func main() { @@ -14,20 +11,16 @@ func main() { if err != nil { panic(err) } - db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucketIfNotExists([]byte(pkg.AccBucket)) - if err != nil { - return fmt.Errorf("create bucket: %s", err) - } - return nil - }) - var client = api.NewClient(api.Params{ - Debug: false, - Host: "http://localhost:8080", - }) ctrl := pkg.ClICtrl{ - Client: client, - DB: db, + Client: api.NewClient(api.Params{ + Debug: false, + Host: "http://localhost:8080", + }), + DB: db, + } + err = ctrl.Init() + if err != nil { + panic(err) } defer func() { if err := db.Close(); err != nil { diff --git a/pkg/controller.go b/pkg/controller.go index c1eb5eb97..d60e1f0dc 100644 --- a/pkg/controller.go +++ b/pkg/controller.go @@ -2,6 +2,7 @@ package pkg import ( "cli-go/internal/api" + "fmt" bolt "go.etcd.io/bbolt" ) @@ -9,3 +10,13 @@ type ClICtrl struct { Client *api.Client DB *bolt.DB } + +func (c *ClICtrl) Init() error { + return c.DB.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucketIfNotExists([]byte(AccBucket)) + if err != nil { + return fmt.Errorf("create bucket: %s", err) + } + return nil + }) +} From a852adae1e1900af06a64b9e5b547d660105b336 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 21 Sep 2023 08:29:02 +0530 Subject: [PATCH 024/183] Write helper method to encrypt --- internal/crypto/crypto.go | 24 ++++++++++++++++++++++-- internal/crypto/crypto_test.go | 2 +- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go index 84c3b2af4..e80700ff2 100644 --- a/internal/crypto/crypto.go +++ b/internal/crypto/crypto.go @@ -45,7 +45,7 @@ func DeriveArgonKey(password, salt string, memLimit, opsLimit int) ([]byte, erro return key, nil } -// decryptChaCha20poly1305 decrypts the given data using the ChaCha20-Poly1305 algorithm. +// DecryptChaCha20poly1305 decrypts the given data using the ChaCha20-Poly1305 algorithm. // Parameters: // - data: The encrypted data as a byte slice. // - key: The key for decryption as a byte slice. @@ -54,7 +54,7 @@ func DeriveArgonKey(password, salt string, memLimit, opsLimit int) ([]byte, erro // Returns: // - A byte slice representing the decrypted data. // - An error object, which is nil if no error occurs. -func decryptChaCha20poly1305(data []byte, key []byte, nonce []byte) ([]byte, error) { +func DecryptChaCha20poly1305(data []byte, key []byte, nonce []byte) ([]byte, error) { reader := bytes.NewReader(data) header := sodium.SecretStreamXCPHeader{Bytes: nonce} decoder, err := sodium.MakeSecretStreamXCPDecoder( @@ -75,6 +75,26 @@ func decryptChaCha20poly1305(data []byte, key []byte, nonce []byte) ([]byte, err return decryptedData[:n], nil } +// EncryptChaCha20poly1305 encrypts the given data using the ChaCha20-Poly1305 algorithm. +// Parameters: +// - data: The plaintext data as a byte slice. +// - key: The key for encryption as a byte slice. +// +// Returns: +// - A byte slice representing the encrypted data. +// - A byte slice representing the header of the encrypted data. +// - An error object, which is nil if no error occurs. +func EncryptChaCha20poly1305(data []byte, key []byte) ([]byte, []byte, error) { + var buf bytes.Buffer + encoder := sodium.MakeSecretStreamXCPEncoder(sodium.SecretStreamXCPKey{Bytes: key}, &buf) + _, err := encoder.WriteAndClose(data) + if err != nil { + log.Println("Failed to write to encoder", err) + return nil, nil, err + } + return buf.Bytes(), encoder.Header().Bytes, nil +} + // DeriveLoginKey derives a login key from the given key encryption key. // This loginKey act as user provided password during SRP authentication. // Parameters: keyEncKey: This is the keyEncryptionKey that is derived from the user's password. diff --git a/internal/crypto/crypto_test.go b/internal/crypto/crypto_test.go index 9f3c60bea..98fd26890 100644 --- a/internal/crypto/crypto_test.go +++ b/internal/crypto/crypto_test.go @@ -43,7 +43,7 @@ func TestDecryptChaCha20poly1305(t *testing.T) { t.Fatalf("Failed to decode cipher nonce: %v", err) } - decryptedText, err := decryptChaCha20poly1305(decodedCipherText, derivedKey, decodedCipherNonce) + decryptedText, err := DecryptChaCha20poly1305(decodedCipherText, derivedKey, decodedCipherNonce) if err != nil { t.Fatalf("Failed to decrypt: %v", err) } From b01c0cecaeec252aca70908a8aa76664af031462 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 21 Sep 2023 08:46:26 +0530 Subject: [PATCH 025/183] Improve email verification flow --- pkg/account.go | 26 ++++++++++++-------------- pkg/sign_in.go | 20 ++++++++++++++++++++ 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/pkg/account.go b/pkg/account.go index 8c8060910..ebff57e5d 100644 --- a/pkg/account.go +++ b/pkg/account.go @@ -4,7 +4,6 @@ import ( "cli-go/internal/api" "context" "encoding/json" - "fmt" "log" bolt "go.etcd.io/bbolt" @@ -38,21 +37,16 @@ func (c *ClICtrl) AddAccount(cxt context.Context) { var authResponse *api.AuthorizationResponse srpAttr, flowErr := c.Client.GetSRPAttributes(cxt, email) if flowErr != nil { - return + // if flowErr type is ApiError and status code is 404, then set verifyEmail to true and continue + // else return + if apiErr, ok := flowErr.(*api.ApiError); ok && apiErr.StatusCode == 404 { + verifyEmail = true + } else { + return + } } if verifyEmail || srpAttr.IsEmailMFAEnabled { - flowErr = c.Client.SendEmailOTP(cxt, email) - if flowErr != nil { - return - } - ott, otpErr := GetCode( - fmt.Sprintf("Enter OTP sent to email %s (or 'c' to cancel)", email), - 6) - if otpErr != nil { - flowErr = otpErr - return - } - authResponse, flowErr = c.Client.VerifyEmail(cxt, email, ott) + authResponse, flowErr = c.validateEmail(cxt, email) } else { authResponse, keyEncKey, flowErr = c.signInViaPassword(cxt, email, srpAttr) } @@ -62,6 +56,10 @@ func (c *ClICtrl) AddAccount(cxt context.Context) { if authResponse.IsMFARequired() { authResponse, flowErr = c.validateTOTP(cxt, authResponse) } + if authResponse.EncryptedToken == "" || authResponse.KeyAttributes == nil { + log.Fatal("no encrypted token or keyAttributes") + return + } if keyEncKey == nil { pass, flowErr := GetSensitiveField("Enter password") if flowErr != nil { diff --git a/pkg/sign_in.go b/pkg/sign_in.go index 7507e8ceb..29b0e5f0d 100644 --- a/pkg/sign_in.go +++ b/pkg/sign_in.go @@ -66,3 +66,23 @@ func (c *ClICtrl) validateTOTP(ctx context.Context, authResp *api.AuthorizationR return totpResp, nil } } + +func (c *ClICtrl) validateEmail(ctx context.Context, email string) (*api.AuthorizationResponse, error) { + err := c.Client.SendEmailOTP(ctx, email) + if err != nil { + return nil, err + } + for { + // CLI prompt for OTP + ott, flowErr := GetCode("Enter OTP", 6) + if flowErr != nil { + return nil, flowErr + } + authResponse, err := c.Client.VerifyEmail(ctx, email, ott) + if err != nil { + log.Printf("failed to verify %v", err) + continue + } + return authResponse, nil + } +} From 383dd7daea8d583d50570b7e3def6f3c6820982e Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 21 Sep 2023 11:55:40 +0530 Subject: [PATCH 026/183] Add support for decrypting masterKey and token --- internal/crypto/crypto.go | 17 +++++++++++ pkg/account.go | 20 +++++++------ pkg/sign_in.go | 63 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 9 deletions(-) diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go index e80700ff2..79111921e 100644 --- a/internal/crypto/crypto.go +++ b/internal/crypto/crypto.go @@ -104,3 +104,20 @@ func DeriveLoginKey(keyEncKey []byte) []byte { // return the first 16 bytes of the derived key return subKey[:16] } + +func SecretBoxOpen(c []byte, n []byte, k []byte) ([]byte, error) { + var cp sodium.Bytes = c + return cp.SecretBoxOpen(sodium.SecretBoxNonce{Bytes: n}, sodium.SecretBoxKey{Bytes: k}) +} + +func SealedBoxOpen(cipherText []byte, publicKey, masterSecret []byte) ([]byte, error) { + var cp sodium.Bytes = cipherText + om, err := cp.SealedBoxOpen(sodium.BoxKP{ + PublicKey: sodium.BoxPublicKey{Bytes: publicKey}, + SecretKey: sodium.BoxSecretKey{Bytes: masterSecret}, + }) + if err != nil { + return nil, fmt.Errorf("failed to open sealed box: %v", err) + } + return om, nil +} diff --git a/pkg/account.go b/pkg/account.go index ebff57e5d..243d48255 100644 --- a/pkg/account.go +++ b/pkg/account.go @@ -4,6 +4,7 @@ import ( "cli-go/internal/api" "context" "encoding/json" + "fmt" "log" bolt "go.etcd.io/bbolt" @@ -32,9 +33,8 @@ func (c *ClICtrl) AddAccount(cxt context.Context) { if flowErr != nil { return } - var keyEncKey []byte var verifyEmail bool - var authResponse *api.AuthorizationResponse + srpAttr, flowErr := c.Client.GetSRPAttributes(cxt, email) if flowErr != nil { // if flowErr type is ApiError and status code is 404, then set verifyEmail to true and continue @@ -45,6 +45,8 @@ func (c *ClICtrl) AddAccount(cxt context.Context) { return } } + var authResponse *api.AuthorizationResponse + var keyEncKey []byte if verifyEmail || srpAttr.IsEmailMFAEnabled { authResponse, flowErr = c.validateEmail(cxt, email) } else { @@ -60,14 +62,14 @@ func (c *ClICtrl) AddAccount(cxt context.Context) { log.Fatal("no encrypted token or keyAttributes") return } - if keyEncKey == nil { - pass, flowErr := GetSensitiveField("Enter password") - if flowErr != nil { - return - } else if pass == "" { - log.Printf("do work") - } + masterKey, token, decErr := c.decryptMasterKeyAndToken(cxt, authResponse, keyEncKey) + if decErr != nil { + flowErr = decErr + return } + // print length + fmt.Printf("master key length: %d\n", len(masterKey)) + fmt.Printf("token length: %d\n", len(token)) } func (c *ClICtrl) ListAccounts(cxt context.Context) error { diff --git a/pkg/sign_in.go b/pkg/sign_in.go index 29b0e5f0d..c9743381b 100644 --- a/pkg/sign_in.go +++ b/pkg/sign_in.go @@ -48,6 +48,69 @@ func (c *ClICtrl) signInViaPassword(ctx context.Context, email string, srpAttr * } } +// Parameters: +// - keyEncKey: key encryption key is derived from user's password. During SRP based login, this key is already derived. +// So, we can pass it to avoid asking for password again. +func (c *ClICtrl) decryptMasterKeyAndToken( + _ context.Context, + authResp *api.AuthorizationResponse, + keyEncKey []byte, +) (masterKey, token []byte, err error) { + + var currentKeyEncKey []byte + for { + if keyEncKey == nil { + // CLI prompt for password + password, flowErr := GetSensitiveField("Enter password") + if flowErr != nil { + return nil, nil, flowErr + } + fmt.Println("\nPlease wait authenticating...") + currentKeyEncKey, err = enteCrypto.DeriveArgonKey(password, + authResp.KeyAttributes.KEKSalt, authResp.KeyAttributes.MemLimit, authResp.KeyAttributes.OpsLimit) + if err != nil { + fmt.Printf("error deriving key encryption key: %v", err) + return nil, nil, err + } + } else { + currentKeyEncKey = keyEncKey + } + + encryptedKey := utils.Base64DecodeString(authResp.KeyAttributes.EncryptedKey) + encryptedKeyNonce := utils.Base64DecodeString(authResp.KeyAttributes.KeyDecryptionNonce) + key, keyErr := enteCrypto.SecretBoxOpen(encryptedKey, encryptedKeyNonce, currentKeyEncKey) + if keyErr != nil { + if keyEncKey != nil { + fmt.Printf("Failed to get key from keyEncryptionKey %s", keyErr) + return nil, nil, keyErr + } else { + fmt.Printf("Incorrect password, error decrypting master key: %v", keyErr) + continue + } + } + masterKey, keyErr = enteCrypto.SecretBoxOpen( + utils.Base64DecodeString(authResp.KeyAttributes.EncryptedSecretKey), + utils.Base64DecodeString(authResp.KeyAttributes.SecretKeyDecryptionNonce), + key, + ) + if keyErr != nil { + fmt.Printf("error decrypting master key: %v", keyErr) + return nil, nil, keyErr + } + token, err = enteCrypto.SealedBoxOpen( + utils.Base64DecodeString(authResp.EncryptedToken), + utils.Base64DecodeString(authResp.KeyAttributes.PublicKey), + masterKey, + ) + if err != nil { + fmt.Printf("error decrypting token: %v", err) + return nil, nil, err + } + break + } + return masterKey, token, nil +} + func (c *ClICtrl) validateTOTP(ctx context.Context, authResp *api.AuthorizationResponse) (*api.AuthorizationResponse, error) { if !authResp.IsMFARequired() { return authResp, nil From f4cc48e1370111112d10843458cb92a5ec85d244 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 21 Sep 2023 12:25:20 +0530 Subject: [PATCH 027/183] Add dependency on zalando/go-keyring --- go.mod | 4 ++++ go.sum | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/go.mod b/go.mod index 34c002048..17f39f5a7 100644 --- a/go.mod +++ b/go.mod @@ -10,8 +10,12 @@ require ( ) require ( + github.com/alessio/shellescape v1.4.1 // indirect + github.com/danieljoos/wincred v1.2.0 // indirect + github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect + github.com/zalando/go-keyring v0.2.3 // indirect ) require ( diff --git a/go.sum b/go.sum index 4a5b4432c..978c21713 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= +github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -47,6 +49,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/danieljoos/wincred v1.2.0 h1:ozqKHaLK0W/ii4KVbbvluM91W2H3Sh0BncbUNPS7jLE= +github.com/danieljoos/wincred v1.2.0/go.mod h1:FzQLLMKBFdvu+osBrnFODiv32YGwCfx0SkRa/eYHgec= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -64,6 +68,8 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -184,6 +190,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zalando/go-keyring v0.2.3 h1:v9CUu9phlABObO4LPWycf+zwMG7nlbb3t/B5wa97yms= +github.com/zalando/go-keyring v0.2.3/go.mod h1:HL4k+OXQfJUWaMnqyuSOc0drfGPX2b51Du6K+MRgZMk= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= From 69a4f4848866100c6284d7b68e226121cb56d4f8 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 21 Sep 2023 15:05:52 +0530 Subject: [PATCH 028/183] Add appType in ctx for client pkg header --- internal/api/client.go | 28 +++++++++++++++++++++------- internal/api/enums.go | 26 ++++++++++++++++++++++++++ pkg/account.go | 17 +++++++++-------- pkg/promt.go | 24 ++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 15 deletions(-) diff --git a/internal/api/client.go b/internal/api/client.go index 5c73866c3..ed82b1c06 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -1,6 +1,7 @@ package api import ( + "context" "github.com/go-resty/resty/v2" ) @@ -8,6 +9,7 @@ const ( EnteAPIEndpoint = "https://api.ente.io" TokenHeader = "X-Auth-Token" TokenQuery = "token" + ClientPkgHeader = "X-Client-Package" ) var ( @@ -25,28 +27,40 @@ type Params struct { Host string } +func readValueFromContext(ctx context.Context, key string) interface{} { + value := ctx.Value(key) + return value +} func NewClient(p Params) *Client { - c := resty.New() + enteAPI := resty.New() if p.Trace { - c.EnableTrace() + enteAPI.EnableTrace() } + enteAPI.OnBeforeRequest(func(c *resty.Client, req *resty.Request) error { + app := readValueFromContext(req.Context(), "app") + if app == nil { + panic("app not set in context") + } + req.Header.Set(ClientPkgHeader, StringToApp(app.(string)).ClientPkg()) + return nil + }) if p.Debug { - c.OnBeforeRequest(func(c *resty.Client, req *resty.Request) error { + enteAPI.OnBeforeRequest(func(c *resty.Client, req *resty.Request) error { logRequest(req) return nil }) - c.OnAfterResponse(func(c *resty.Client, resp *resty.Response) error { + enteAPI.OnAfterResponse(func(c *resty.Client, resp *resty.Response) error { logResponse(resp) return nil }) } if p.Host != "" { - c.SetBaseURL(p.Host) + enteAPI.SetBaseURL(p.Host) } else { - c.SetBaseURL(EnteAPIEndpoint) + enteAPI.SetBaseURL(EnteAPIEndpoint) } return &Client{ - restClient: c, + restClient: enteAPI, } } diff --git a/internal/api/enums.go b/internal/api/enums.go index 42cddc287..eaaaf010d 100644 --- a/internal/api/enums.go +++ b/internal/api/enums.go @@ -1,5 +1,7 @@ package api +import "fmt" + type App string const ( @@ -7,3 +9,27 @@ const ( AppAuth App = "auth" AppLocker App = "locker" ) + +func StringToApp(s string) App { + switch s { + case "photos": + return AppPhotos + case "auth": + return AppAuth + case "locker": + return AppLocker + default: + panic(fmt.Sprintf("invalid app: %s", s)) + } +} +func (a App) ClientPkg() string { + switch a { + case AppPhotos: + return "io.ente.photos" + case AppAuth: + return "io.ente.auth" + case AppLocker: + return "io.ente.locker" + } + return "" +} diff --git a/pkg/account.go b/pkg/account.go index 243d48255..2809edf5d 100644 --- a/pkg/account.go +++ b/pkg/account.go @@ -2,33 +2,34 @@ package pkg import ( "cli-go/internal/api" + "cli-go/pkg/model" "context" "encoding/json" "fmt" "log" + "runtime" bolt "go.etcd.io/bbolt" ) const AccBucket = "accounts" -type AccountInfo struct { - Email string `json:"email" binding:"required"` - UserID int64 `json:"userID" binding:"required"` - App api.App `json:"app" binding:"required"` -} - func (c *ClICtrl) AddAccount(cxt context.Context) { var flowErr error defer func() { if r := recover(); r != nil { - log.Println("recovered from panic", r) + fmt.Println("Panic occurred:", r) + // Print the stack trace + stackTrace := make([]byte, 1024*8) + stackTrace = stackTrace[:runtime.Stack(stackTrace, false)] + fmt.Printf("Stack Trace:\n%s", stackTrace) } if flowErr != nil { log.Fatal(flowErr) } }() - + app := GetAppType() + cxt = context.WithValue(cxt, "app", string(app)) email, flowErr := GetUserInput("Enter email address") if flowErr != nil { return diff --git a/pkg/promt.go b/pkg/promt.go index 32936f469..65abb0b09 100644 --- a/pkg/promt.go +++ b/pkg/promt.go @@ -1,6 +1,7 @@ package pkg import ( + "cli-go/internal/api" "errors" "fmt" "log" @@ -31,6 +32,29 @@ func GetUserInput(label string) (string, error) { return input, nil } +func GetAppType() api.App { + for { + app, err := GetUserInput("Enter app type (default: photos)") + if err != nil { + fmt.Printf("Use default app type: %s\n", api.AppPhotos) + return api.AppPhotos + } + switch app { + case "photos": + return api.AppPhotos + case "auth": + return api.AppAuth + case "locker": + return api.AppLocker + case "": + return api.AppPhotos + default: + fmt.Println("invalid app type") + continue + } + } +} + func GetCode(promptText string, length int) (string, error) { for { ott, err := GetUserInput(promptText) From 2c9b0ea7a9d6a82fd4b598c745104312420bb03d Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 21 Sep 2023 15:23:37 +0530 Subject: [PATCH 029/183] Add common recover with logs --- cmd/account.go | 3 ++- cmd/root.go | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/cmd/account.go b/cmd/account.go index 454d966a6..0ea414179 100644 --- a/cmd/account.go +++ b/cmd/account.go @@ -17,6 +17,7 @@ var listAccCmd = &cobra.Command{ Use: "list", Short: "list configured accounts", RunE: func(cmd *cobra.Command, args []string) error { + recoverWithLog() return ctrl.ListAccounts(context.Background()) }, } @@ -26,6 +27,7 @@ var addAccCmd = &cobra.Command{ Use: "add", Short: "Add a new account", Run: func(cmd *cobra.Command, args []string) { + recoverWithLog() ctrl.AddAccount(context.Background()) }, } @@ -34,7 +36,6 @@ func init() { // Add 'config' subcommands to the root command rootCmd.AddCommand(accountCmd) - // Add 'config' subcommands to the 'config' command accountCmd.AddCommand(listAccCmd, addAccCmd) } diff --git a/cmd/root.go b/cmd/root.go index 0b30ea92f..8001465ba 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -4,6 +4,7 @@ import ( "cli-go/pkg" "fmt" "os" + "runtime" "github.com/spf13/viper" @@ -50,3 +51,13 @@ func init() { viper.AddConfigPath(".") // Search for config file in the current directory viper.ReadInConfig() // Read the configuration file if it exists } + +func recoverWithLog() { + if r := recover(); r != nil { + fmt.Println("Panic occurred:", r) + // Print the stack trace + stackTrace := make([]byte, 1024*8) + stackTrace = stackTrace[:runtime.Stack(stackTrace, false)] + fmt.Printf("Stack Trace:\n%s", stackTrace) + } +} From 1b45bc0e0c7044d23830028ced39572a81514452 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 21 Sep 2023 17:37:18 +0530 Subject: [PATCH 030/183] Add support for storing accounts --- cmd/account.go | 1 - internal/api/client.go | 1 + pkg/account.go | 60 ++++++++++++++++++++++++++++------------- pkg/model/account.go | 11 ++++++++ pkg/model/enc_string.go | 30 +++++++++++++++++++++ pkg/secret.go | 30 +++++++++++++++++++++ 6 files changed, 113 insertions(+), 20 deletions(-) create mode 100644 pkg/model/account.go create mode 100644 pkg/model/enc_string.go create mode 100644 pkg/secret.go diff --git a/cmd/account.go b/cmd/account.go index 0ea414179..0a1e30f87 100644 --- a/cmd/account.go +++ b/cmd/account.go @@ -33,7 +33,6 @@ var addAccCmd = &cobra.Command{ } func init() { - // Add 'config' subcommands to the root command rootCmd.AddCommand(accountCmd) // Add 'config' subcommands to the 'config' command diff --git a/internal/api/client.go b/internal/api/client.go index ed82b1c06..fde874bd4 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -31,6 +31,7 @@ func readValueFromContext(ctx context.Context, key string) interface{} { value := ctx.Value(key) return value } + func NewClient(p Params) *Client { enteAPI := resty.New() if p.Trace { diff --git a/pkg/account.go b/pkg/account.go index 2809edf5d..324aff557 100644 --- a/pkg/account.go +++ b/pkg/account.go @@ -7,7 +7,6 @@ import ( "encoding/json" "fmt" "log" - "runtime" bolt "go.etcd.io/bbolt" ) @@ -15,15 +14,9 @@ import ( const AccBucket = "accounts" func (c *ClICtrl) AddAccount(cxt context.Context) { + var flowErr error defer func() { - if r := recover(); r != nil { - fmt.Println("Panic occurred:", r) - // Print the stack trace - stackTrace := make([]byte, 1024*8) - stackTrace = stackTrace[:runtime.Stack(stackTrace, false)] - fmt.Printf("Stack Trace:\n%s", stackTrace) - } if flowErr != nil { log.Fatal(flowErr) } @@ -60,42 +53,71 @@ func (c *ClICtrl) AddAccount(cxt context.Context) { authResponse, flowErr = c.validateTOTP(cxt, authResponse) } if authResponse.EncryptedToken == "" || authResponse.KeyAttributes == nil { - log.Fatal("no encrypted token or keyAttributes") - return + panic("no encrypted token or keyAttributes") } masterKey, token, decErr := c.decryptMasterKeyAndToken(cxt, authResponse, keyEncKey) if decErr != nil { flowErr = decErr return } - // print length - fmt.Printf("master key length: %d\n", len(masterKey)) - fmt.Printf("token length: %d\n", len(token)) + err := c.storeAccount(cxt, email, authResponse.ID, app, masterKey, token) + if err != nil { + flowErr = err + return + } else { + fmt.Println("Account added successfully") + } +} + +func (c *ClICtrl) storeAccount(ctx context.Context, email string, userID int64, app api.App, masterKey, token []byte) error { + // get password + secret := GetOrCreateClISecret() + err := c.DB.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucketIfNotExists([]byte(AccBucket)) + if err != nil { + return err + } + accInfo := model.AccountInfo{ + Email: email, + UserID: userID, + MasterKey: *model.MakeEncString(string(masterKey), secret), + Token: *model.MakeEncString(string(token), secret), + App: app, + } + accInfoBytes, err := json.Marshal(accInfo) + if err != nil { + return err + } + accountKey := fmt.Sprintf("%s-%d", app, userID) + return b.Put([]byte(accountKey), accInfoBytes) + }) + return err } func (c *ClICtrl) ListAccounts(cxt context.Context) error { err := c.DB.View(func(tx *bolt.Tx) error { b := tx.Bucket([]byte(AccBucket)) - log.Printf("total accounts: %d", b.Stats().KeyN) + fmt.Printf("Configured accounts: %d\n", b.Stats().KeyN) err := b.ForEach(func(k, v []byte) error { - var info AccountInfo + var info model.AccountInfo err := json.Unmarshal(v, &info) if err != nil { return err } - log.Println(info) + fmt.Println("====================================") + fmt.Println("Email: ", info.Email) + fmt.Println("UserID: ", info.UserID) + fmt.Println("App: ", info.App) + fmt.Println("====================================") return nil }) if err != nil { - log.Fatal("error listing accounts", err) return err } return nil }) if err != nil { - log.Fatal("error listing accounts", err) return err } return err - } diff --git a/pkg/model/account.go b/pkg/model/account.go new file mode 100644 index 000000000..ab45adde1 --- /dev/null +++ b/pkg/model/account.go @@ -0,0 +1,11 @@ +package model + +import "cli-go/internal/api" + +type AccountInfo struct { + Email string `json:"email" binding:"required"` + UserID int64 `json:"userID" binding:"required"` + App api.App `json:"app" binding:"required"` + MasterKey EncString `json:"masterKey" binding:"required"` + Token EncString `json:"token" binding:"required"` +} diff --git a/pkg/model/enc_string.go b/pkg/model/enc_string.go new file mode 100644 index 000000000..d6ec41074 --- /dev/null +++ b/pkg/model/enc_string.go @@ -0,0 +1,30 @@ +package model + +import ( + "cli-go/internal/crypto" + "log" +) + +type EncString struct { + CipherText string `json:"cipherText"` + Nonce string `json:"nonce"` +} + +func MakeEncString(plainText string, key []byte) *EncString { + cipher, nonce, err := crypto.EncryptChaCha20poly1305([]byte(plainText), key) + if err != nil { + log.Fatalf("failed to encrypt %s", err) + } + return &EncString{ + CipherText: string(cipher), + Nonce: string(nonce), + } +} + +func (e *EncString) MustDecrypt(key []byte) string { + plainBytes, err := crypto.DecryptChaCha20poly1305([]byte(e.CipherText), key, []byte(e.Nonce)) + if err != nil { + panic(err) + } + return string(plainBytes) +} diff --git a/pkg/secret.go b/pkg/secret.go new file mode 100644 index 000000000..476f0fe67 --- /dev/null +++ b/pkg/secret.go @@ -0,0 +1,30 @@ +package pkg + +import ( + "crypto/rand" + "errors" + "fmt" + "github.com/zalando/go-keyring" + "log" +) + +func GetOrCreateClISecret() []byte { + // get password + secret, err := keyring.Get("ente-cli-cli", "ghost") + if err != nil { + if !errors.Is(err, keyring.ErrNotFound) { + log.Fatal(fmt.Errorf("error getting password from keyring: %w", err)) + } + key := make([]byte, 32) + _, err = rand.Read(key) + if err != nil { + // handle error here + } + keySetErr := keyring.Set("ente-cli-cli", "ghost", string(key)) + if keySetErr != nil { + log.Fatal(fmt.Errorf("error setting password in keyring: %w", keySetErr)) + } + secret = string(key) + } + return []byte(secret) +} From 9942fd764e8f7ba453e87af9211449e5f488fce1 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 21 Sep 2023 17:40:35 +0530 Subject: [PATCH 031/183] Update README.md --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 7af30add8..8d2e1b7bf 100644 --- a/README.md +++ b/README.md @@ -18,3 +18,17 @@ or you can run the following command ./bin/ente-cli --help ``` +### Getting Started + +#### Accounts + +* Add an account + ```shell + ente-cli account add + ``` + +* List accounts + ```shell + ente-cli account list + ``` + From abdc0d262d74db8a25c02700f431a84f4e89cdf9 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 21 Sep 2023 18:10:18 +0530 Subject: [PATCH 032/183] Add dockerfile --- Dockerfile | 26 ++++++++++++++++++++++++++ README.md | 38 +++++++++++++++++++++++++------------- 2 files changed, 51 insertions(+), 13 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..24ab96737 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM golang:1.20-alpine3.17 as builder +RUN apk add --no-cache gcc musl-dev git build-base pkgconfig libsodium-dev + +ENV GOOS=linux + +WORKDIR /etc/ente/ + +COPY go.mod . +COPY go.sum . +RUN go mod download + +COPY . . +RUN --mount=type=cache,target=/root/.cache/go-build \ + go build -o ente-cli main.go + +FROM alpine:3.17 +RUN apk add libsodium-dev +COPY --from=builder /etc/ente/ente-cli . +#COPY configurations configurations +#COPY migrations migrations +#COPY mail-templates mail-templates + +ARG GIT_COMMIT +ENV GIT_COMMIT=$GIT_COMMIT + +CMD ["./ente-cli"] diff --git a/README.md b/README.md index 8d2e1b7bf..b0fe07ad5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,21 @@ # cli tool for exporting ente photos +### Getting Started + +#### Accounts + +* Add an account + ```shell + ente-cli account add + ``` + +* List accounts + ```shell + ente-cli account list + ``` + + + ## Testing Run the release script to build the binary and run it. @@ -18,17 +34,13 @@ or you can run the following command ./bin/ente-cli --help ``` -### Getting Started - -#### Accounts - -* Add an account - ```shell - ente-cli account add - ``` - -* List accounts - ```shell - ente-cli account list - ``` +## Docker + Build the docker image + ```shell + docker build -t ente-cli:latest . + ``` + Run the commands using: + ```shell + docker run -it --rm ente-cli:latest ./ente-cli --help + ``` From 49586cae3c6007de1ffe3aab84dbfc3434b6f8b4 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 22 Sep 2023 10:05:25 +0530 Subject: [PATCH 033/183] Add GetListAccounts + refactor --- pkg/account.go | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/pkg/account.go b/pkg/account.go index 324aff557..666dfa184 100644 --- a/pkg/account.go +++ b/pkg/account.go @@ -14,7 +14,6 @@ import ( const AccBucket = "accounts" func (c *ClICtrl) AddAccount(cxt context.Context) { - var flowErr error defer func() { if flowErr != nil { @@ -69,7 +68,7 @@ func (c *ClICtrl) AddAccount(cxt context.Context) { } } -func (c *ClICtrl) storeAccount(ctx context.Context, email string, userID int64, app api.App, masterKey, token []byte) error { +func (c *ClICtrl) storeAccount(_ context.Context, email string, userID int64, app api.App, masterKey, token []byte) error { // get password secret := GetOrCreateClISecret() err := c.DB.Update(func(tx *bolt.Tx) error { @@ -94,21 +93,17 @@ func (c *ClICtrl) storeAccount(ctx context.Context, email string, userID int64, return err } -func (c *ClICtrl) ListAccounts(cxt context.Context) error { +func (c *ClICtrl) GetAccounts(cxt context.Context) ([]model.AccountInfo, error) { + var accounts []model.AccountInfo err := c.DB.View(func(tx *bolt.Tx) error { b := tx.Bucket([]byte(AccBucket)) - fmt.Printf("Configured accounts: %d\n", b.Stats().KeyN) err := b.ForEach(func(k, v []byte) error { var info model.AccountInfo err := json.Unmarshal(v, &info) if err != nil { return err } - fmt.Println("====================================") - fmt.Println("Email: ", info.Email) - fmt.Println("UserID: ", info.UserID) - fmt.Println("App: ", info.App) - fmt.Println("====================================") + accounts = append(accounts, info) return nil }) if err != nil { @@ -116,8 +111,21 @@ func (c *ClICtrl) ListAccounts(cxt context.Context) error { } return nil }) + return accounts, err +} + +func (c *ClICtrl) ListAccounts(cxt context.Context) error { + accounts, err := c.GetAccounts(cxt) if err != nil { return err } - return err + fmt.Printf("Configured accounts: %d\n", len(accounts)) + for _, acc := range accounts { + fmt.Println("====================================") + fmt.Println("Email: ", acc.Email) + fmt.Println("UserID: ", acc.UserID) + fmt.Println("App: ", acc.App) + fmt.Println("====================================") + } + return nil } From 107fd736fd1b1efabddb2e59d31dbcdd6b116e1b Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 22 Sep 2023 10:14:22 +0530 Subject: [PATCH 034/183] Create various buckets for an account --- pkg/model/account.go | 13 ++++++++++++- pkg/remote_sync.go | 31 +++++++++++++++++++++++++++++++ pkg/sync.go | 17 +++++++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 pkg/remote_sync.go create mode 100644 pkg/sync.go diff --git a/pkg/model/account.go b/pkg/model/account.go index ab45adde1..260b5ca4a 100644 --- a/pkg/model/account.go +++ b/pkg/model/account.go @@ -1,6 +1,9 @@ package model -import "cli-go/internal/api" +import ( + "cli-go/internal/api" + "fmt" +) type AccountInfo struct { Email string `json:"email" binding:"required"` @@ -9,3 +12,11 @@ type AccountInfo struct { MasterKey EncString `json:"masterKey" binding:"required"` Token EncString `json:"token" binding:"required"` } + +func (a AccountInfo) AccountKey() string { + return fmt.Sprintf("%s-%d", a.App, a.UserID) +} + +func (a AccountInfo) DataBucket() string { + return fmt.Sprintf("%s-%d-data", a.App, a.UserID) +} diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go new file mode 100644 index 000000000..add3a6922 --- /dev/null +++ b/pkg/remote_sync.go @@ -0,0 +1,31 @@ +package pkg + +import ( + "cli-go/pkg/model" + "fmt" + bolt "go.etcd.io/bbolt" + "log" +) + +func (c *ClICtrl) SyncAccount(account model.AccountInfo) error { + return createDataBuckets(c.DB, account) +} + +var dataCategories = []string{"remote-collections", "local-collections", "remote-files", "local-files", "remote-collection-removed", "remote-files-removed"} + +func createDataBuckets(db *bolt.DB, account model.AccountInfo) error { + return db.Update(func(tx *bolt.Tx) error { + dataBucket, err := tx.CreateBucketIfNotExists([]byte(account.DataBucket())) + if err != nil { + return fmt.Errorf("create bucket: %s", err) + } + for _, category := range dataCategories { + exists, err := dataBucket.CreateBucketIfNotExists([]byte(fmt.Sprintf(category))) + if err != nil { + return err + } + log.Println("SubBucket for category", category, "with parent", account.DataBucket(), "exists:", exists) + } + return nil + }) +} diff --git a/pkg/sync.go b/pkg/sync.go new file mode 100644 index 000000000..1928bab84 --- /dev/null +++ b/pkg/sync.go @@ -0,0 +1,17 @@ +package pkg + +import "context" + +func (c *ClICtrl) StartSync() error { + accounts, err := c.GetAccounts(context.Background()) + if err != nil { + return err + } + for _, account := range accounts { + err = c.SyncAccount(account) + if err != nil { + return err + } + } + return nil +} From 235face77179353c88c0c576ec8ea028b92a8a5d Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 22 Sep 2023 10:50:23 +0530 Subject: [PATCH 035/183] Test for enc_string --- pkg/model/enc_string_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 pkg/model/enc_string_test.go diff --git a/pkg/model/enc_string_test.go b/pkg/model/enc_string_test.go new file mode 100644 index 000000000..2b3866ecc --- /dev/null +++ b/pkg/model/enc_string_test.go @@ -0,0 +1,20 @@ +package model + +import ( + "crypto/rand" + "testing" +) + +func TestEncString(t *testing.T) { + key := make([]byte, 32) + _, err := rand.Read(key) + if err != nil { + t.Fatalf("error generating key: %v", err) + } + data := "dataToEncrypt" + encData := MakeEncString(data, key) + decryptedData := encData.MustDecrypt(key) + if decryptedData != data { + t.Fatalf("decrypted data is not equal to original data") + } +} From 490e145053f1830d0cb85575d742f4a77451c511 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 22 Sep 2023 10:55:33 +0530 Subject: [PATCH 036/183] Handle err + minor refactor --- pkg/secret.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/secret.go b/pkg/secret.go index 476f0fe67..3d440863d 100644 --- a/pkg/secret.go +++ b/pkg/secret.go @@ -4,8 +4,9 @@ import ( "crypto/rand" "errors" "fmt" - "github.com/zalando/go-keyring" "log" + + "github.com/zalando/go-keyring" ) func GetOrCreateClISecret() []byte { @@ -18,13 +19,14 @@ func GetOrCreateClISecret() []byte { key := make([]byte, 32) _, err = rand.Read(key) if err != nil { - // handle error here + log.Fatal(fmt.Errorf("error generating key: %w", err)) } - keySetErr := keyring.Set("ente-cli-cli", "ghost", string(key)) + secret = string(key) + keySetErr := keyring.Set("ente-cli-cli", "ghost", string(secret)) if keySetErr != nil { log.Fatal(fmt.Errorf("error setting password in keyring: %w", keySetErr)) } - secret = string(key) + } return []byte(secret) } From 40acd1d22640c947995931ba4978982295d029e2 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 22 Sep 2023 10:57:58 +0530 Subject: [PATCH 037/183] Use token map for storing multiple tokens --- internal/api/client.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/api/client.go b/internal/api/client.go index fde874bd4..9b86a590e 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -18,7 +18,7 @@ var ( type Client struct { restClient *resty.Client - authToken *string + tokenMap map[string]string } type Params struct { @@ -63,5 +63,10 @@ func NewClient(p Params) *Client { } return &Client{ restClient: enteAPI, + tokenMap: make(map[string]string), } } + +func (c *Client) AddToken(id string, token string) { + c.tokenMap[id] = token +} From 1e519280edbf445973cd1a4d2b48b32d03baa1db Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 22 Sep 2023 10:59:02 +0530 Subject: [PATCH 038/183] Rename --- pkg/account.go | 14 +++++++------- pkg/model/account.go | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/account.go b/pkg/account.go index 666dfa184..838fc3862 100644 --- a/pkg/account.go +++ b/pkg/account.go @@ -76,7 +76,7 @@ func (c *ClICtrl) storeAccount(_ context.Context, email string, userID int64, ap if err != nil { return err } - accInfo := model.AccountInfo{ + accInfo := model.Account{ Email: email, UserID: userID, MasterKey: *model.MakeEncString(string(masterKey), secret), @@ -87,18 +87,18 @@ func (c *ClICtrl) storeAccount(_ context.Context, email string, userID int64, ap if err != nil { return err } - accountKey := fmt.Sprintf("%s-%d", app, userID) + accountKey := accInfo.AccountKey() return b.Put([]byte(accountKey), accInfoBytes) }) return err } -func (c *ClICtrl) GetAccounts(cxt context.Context) ([]model.AccountInfo, error) { - var accounts []model.AccountInfo +func (c *ClICtrl) GetAccounts(cxt context.Context) ([]model.Account, error) { + var accounts []model.Account err := c.DB.View(func(tx *bolt.Tx) error { b := tx.Bucket([]byte(AccBucket)) err := b.ForEach(func(k, v []byte) error { - var info model.AccountInfo + var info model.Account err := json.Unmarshal(v, &info) if err != nil { return err @@ -123,8 +123,8 @@ func (c *ClICtrl) ListAccounts(cxt context.Context) error { for _, acc := range accounts { fmt.Println("====================================") fmt.Println("Email: ", acc.Email) - fmt.Println("UserID: ", acc.UserID) - fmt.Println("App: ", acc.App) + fmt.Println("ID: ", acc.UserID) + fmt.Println("App: ", acc.App) fmt.Println("====================================") } return nil diff --git a/pkg/model/account.go b/pkg/model/account.go index 260b5ca4a..f251a1f50 100644 --- a/pkg/model/account.go +++ b/pkg/model/account.go @@ -5,7 +5,7 @@ import ( "fmt" ) -type AccountInfo struct { +type Account struct { Email string `json:"email" binding:"required"` UserID int64 `json:"userID" binding:"required"` App api.App `json:"app" binding:"required"` @@ -13,10 +13,10 @@ type AccountInfo struct { Token EncString `json:"token" binding:"required"` } -func (a AccountInfo) AccountKey() string { +func (a Account) AccountKey() string { return fmt.Sprintf("%s-%d", a.App, a.UserID) } -func (a AccountInfo) DataBucket() string { +func (a Account) DataBucket() string { return fmt.Sprintf("%s-%d-data", a.App, a.UserID) } From 8a26f13b76b2456a8aba2989b781a49278ba5013 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 22 Sep 2023 13:47:40 +0530 Subject: [PATCH 039/183] EncString: Use base64 encoding --- pkg/model/enc_string.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/model/enc_string.go b/pkg/model/enc_string.go index d6ec41074..5187a04a5 100644 --- a/pkg/model/enc_string.go +++ b/pkg/model/enc_string.go @@ -2,6 +2,7 @@ package model import ( "cli-go/internal/crypto" + "cli-go/utils" "log" ) @@ -16,15 +17,15 @@ func MakeEncString(plainText string, key []byte) *EncString { log.Fatalf("failed to encrypt %s", err) } return &EncString{ - CipherText: string(cipher), - Nonce: string(nonce), + CipherText: utils.BytesToBase64(cipher), + Nonce: utils.BytesToBase64(nonce), } } func (e *EncString) MustDecrypt(key []byte) string { - plainBytes, err := crypto.DecryptChaCha20poly1305([]byte(e.CipherText), key, []byte(e.Nonce)) + plainBytes, err := crypto.DecryptChaCha20poly1305(utils.Base64DecodeString(e.CipherText), key, utils.Base64DecodeString(e.Nonce)) if err != nil { panic(err) } - return string(plainBytes) + return utils.BytesToBase64(plainBytes) } From 42112309d5ea6ccd8d408e5293af6d76061d00d3 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 22 Sep 2023 13:48:33 +0530 Subject: [PATCH 040/183] RestClient: Attach account specific token --- internal/api/client.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/api/client.go b/internal/api/client.go index 9b86a590e..36162e8ea 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -15,6 +15,7 @@ const ( var ( RedactedHeaders = []string{TokenHeader, " X-Request-Id"} ) +var tokenMap map[string]string = make(map[string]string) type Client struct { restClient *resty.Client @@ -43,6 +44,12 @@ func NewClient(p Params) *Client { panic("app not set in context") } req.Header.Set(ClientPkgHeader, StringToApp(app.(string)).ClientPkg()) + accountId := readValueFromContext(req.Context(), "account_id") + if accountId != nil && accountId != "" { + if token, ok := tokenMap[accountId.(string)]; ok { + req.SetHeader(TokenHeader, token) + } + } return nil }) if p.Debug { @@ -63,10 +70,9 @@ func NewClient(p Params) *Client { } return &Client{ restClient: enteAPI, - tokenMap: make(map[string]string), } } func (c *Client) AddToken(id string, token string) { - c.tokenMap[id] = token + tokenMap[id] = token } From e3c6f4c8a7bc26643cd099e8ed21cfbee0ab97b4 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 22 Sep 2023 13:49:09 +0530 Subject: [PATCH 041/183] API: Fix GetCollection --- internal/api/collection.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/internal/api/collection.go b/internal/api/collection.go index 597724981..2b24145a6 100644 --- a/internal/api/collection.go +++ b/internal/api/collection.go @@ -3,11 +3,20 @@ package api import "context" func (c *Client) GetCollections(ctx context.Context, sinceTime int) ([]Collection, error) { - var collections []Collection - _, err := c.restClient.R(). + var res struct { + Collections []Collection `json:"collections"` + } + r, err := c.restClient.R(). SetContext(ctx). - SetQueryParam("since", string(sinceTime)). - SetResult(&collections). + SetQueryParam("since", "0"). + SetResult(&res). Get("/collections") - return collections, err + if r.IsError() { + return nil, &ApiError{ + StatusCode: r.StatusCode(), + Message: r.String(), + } + } + + return res.Collections, err } From b4f0994a349b8961c7545de234debbdd2072d5fe Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 22 Sep 2023 13:50:50 +0530 Subject: [PATCH 042/183] Fetch remote collection & print ids --- internal/api/client.go | 1 - pkg/remote_sync.go | 40 +++++++++++++++++++++++++++++++++++----- pkg/sync.go | 11 ++++++++++- 3 files changed, 45 insertions(+), 7 deletions(-) diff --git a/internal/api/client.go b/internal/api/client.go index 36162e8ea..7cd28c430 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -19,7 +19,6 @@ var tokenMap map[string]string = make(map[string]string) type Client struct { restClient *resty.Client - tokenMap map[string]string } type Params struct { diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index add3a6922..843cc7bc2 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -2,30 +2,60 @@ package pkg import ( "cli-go/pkg/model" + "cli-go/utils" + "context" + "encoding/base64" "fmt" bolt "go.etcd.io/bbolt" "log" ) -func (c *ClICtrl) SyncAccount(account model.AccountInfo) error { - return createDataBuckets(c.DB, account) +func (c *ClICtrl) SyncAccount(account model.Account) error { + log.SetPrefix(fmt.Sprintf("[%s] ", account.Email)) + cliSecret := GetOrCreateClISecret() + err := createDataBuckets(c.DB, account) + if err != nil { + return err + } + token := account.Token.MustDecrypt(cliSecret) + urlEncodedToken := base64.URLEncoding.EncodeToString(utils.Base64DecodeString(token)) + c.Client.AddToken(account.AccountKey(), urlEncodedToken) + ctx := c.GetRequestContext(context.Background(), account) + return c.syncRemoteCollections(ctx, account) +} + +func (c *ClICtrl) GetRequestContext(ctx context.Context, account model.Account) context.Context { + ctx = context.WithValue(ctx, "app", string(account.App)) + ctx = context.WithValue(ctx, "account_id", account.AccountKey()) + return ctx } var dataCategories = []string{"remote-collections", "local-collections", "remote-files", "local-files", "remote-collection-removed", "remote-files-removed"} -func createDataBuckets(db *bolt.DB, account model.AccountInfo) error { +func createDataBuckets(db *bolt.DB, account model.Account) error { return db.Update(func(tx *bolt.Tx) error { dataBucket, err := tx.CreateBucketIfNotExists([]byte(account.DataBucket())) if err != nil { return fmt.Errorf("create bucket: %s", err) } for _, category := range dataCategories { - exists, err := dataBucket.CreateBucketIfNotExists([]byte(fmt.Sprintf(category))) + _, err := dataBucket.CreateBucketIfNotExists([]byte(fmt.Sprintf(category))) if err != nil { return err } - log.Println("SubBucket for category", category, "with parent", account.DataBucket(), "exists:", exists) } return nil }) } + +func (c *ClICtrl) syncRemoteCollections(ctx context.Context, info model.Account) error { + collections, err := c.Client.GetCollections(ctx, 0) + if err != nil { + log.Printf("failed to get collections: %s\n", err) + return err + } + for _, collection := range collections { + fmt.Printf("Collection %d\n", collection.ID) + } + return nil +} diff --git a/pkg/sync.go b/pkg/sync.go index 1928bab84..3476d2e8e 100644 --- a/pkg/sync.go +++ b/pkg/sync.go @@ -1,15 +1,24 @@ package pkg -import "context" +import ( + "context" + "fmt" +) func (c *ClICtrl) StartSync() error { accounts, err := c.GetAccounts(context.Background()) if err != nil { return err } + if len(accounts) == 0 { + fmt.Printf("No accounts to sync\n") + return nil + } for _, account := range accounts { + fmt.Printf("Syncing account %s\n", account.Email) err = c.SyncAccount(account) if err != nil { + fmt.Printf("Error syncing account %s: %s\n", account.Email, err) return err } } From 652cf53e32d648e3c31637adec528d1dd3d107ea Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 22 Sep 2023 17:48:14 +0530 Subject: [PATCH 043/183] Store both masterKey and secretKey --- pkg/account.go | 11 +++++---- pkg/model/account.go | 1 + pkg/model/enc_string.go | 8 +++---- pkg/remote_sync.go | 2 +- pkg/sign_in.go | 52 +++++++++++++++++++++++++---------------- utils/encoding.go | 4 ++-- 6 files changed, 46 insertions(+), 32 deletions(-) diff --git a/pkg/account.go b/pkg/account.go index 838fc3862..89d2f435c 100644 --- a/pkg/account.go +++ b/pkg/account.go @@ -54,12 +54,12 @@ func (c *ClICtrl) AddAccount(cxt context.Context) { if authResponse.EncryptedToken == "" || authResponse.KeyAttributes == nil { panic("no encrypted token or keyAttributes") } - masterKey, token, decErr := c.decryptMasterKeyAndToken(cxt, authResponse, keyEncKey) + secretInfo, decErr := c.decryptMasterKeyAndToken(cxt, authResponse, keyEncKey) if decErr != nil { flowErr = decErr return } - err := c.storeAccount(cxt, email, authResponse.ID, app, masterKey, token) + err := c.storeAccount(cxt, email, authResponse.ID, app, secretInfo) if err != nil { flowErr = err return @@ -68,7 +68,7 @@ func (c *ClICtrl) AddAccount(cxt context.Context) { } } -func (c *ClICtrl) storeAccount(_ context.Context, email string, userID int64, app api.App, masterKey, token []byte) error { +func (c *ClICtrl) storeAccount(_ context.Context, email string, userID int64, app api.App, secretInfo *accSecretInfo) error { // get password secret := GetOrCreateClISecret() err := c.DB.Update(func(tx *bolt.Tx) error { @@ -79,8 +79,9 @@ func (c *ClICtrl) storeAccount(_ context.Context, email string, userID int64, ap accInfo := model.Account{ Email: email, UserID: userID, - MasterKey: *model.MakeEncString(string(masterKey), secret), - Token: *model.MakeEncString(string(token), secret), + MasterKey: *model.MakeEncString(string(secretInfo.MasterKey), secret), + Token: *model.MakeEncString(string(secretInfo.Token), secret), + SecretKey: *model.MakeEncString(string(secretInfo.SecretKey), secret), App: app, } accInfoBytes, err := json.Marshal(accInfo) diff --git a/pkg/model/account.go b/pkg/model/account.go index f251a1f50..9031b95fa 100644 --- a/pkg/model/account.go +++ b/pkg/model/account.go @@ -10,6 +10,7 @@ type Account struct { UserID int64 `json:"userID" binding:"required"` App api.App `json:"app" binding:"required"` MasterKey EncString `json:"masterKey" binding:"required"` + SecretKey EncString `json:"secretKey" binding:"required"` Token EncString `json:"token" binding:"required"` } diff --git a/pkg/model/enc_string.go b/pkg/model/enc_string.go index 5187a04a5..d31c86e12 100644 --- a/pkg/model/enc_string.go +++ b/pkg/model/enc_string.go @@ -17,15 +17,15 @@ func MakeEncString(plainText string, key []byte) *EncString { log.Fatalf("failed to encrypt %s", err) } return &EncString{ - CipherText: utils.BytesToBase64(cipher), - Nonce: utils.BytesToBase64(nonce), + CipherText: utils.EncodeBase64(cipher), + Nonce: utils.EncodeBase64(nonce), } } func (e *EncString) MustDecrypt(key []byte) string { - plainBytes, err := crypto.DecryptChaCha20poly1305(utils.Base64DecodeString(e.CipherText), key, utils.Base64DecodeString(e.Nonce)) + plainBytes, err := crypto.DecryptChaCha20poly1305(utils.DecodeBase64(e.CipherText), key, utils.DecodeBase64(e.Nonce)) if err != nil { panic(err) } - return utils.BytesToBase64(plainBytes) + return utils.EncodeBase64(plainBytes) } diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index 843cc7bc2..52f6741e5 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -18,7 +18,7 @@ func (c *ClICtrl) SyncAccount(account model.Account) error { return err } token := account.Token.MustDecrypt(cliSecret) - urlEncodedToken := base64.URLEncoding.EncodeToString(utils.Base64DecodeString(token)) + urlEncodedToken := base64.URLEncoding.EncodeToString(utils.DecodeBase64(token)) c.Client.AddToken(account.AccountKey(), urlEncodedToken) ctx := c.GetRequestContext(context.Background(), account) return c.syncRemoteCollections(ctx, account) diff --git a/pkg/sign_in.go b/pkg/sign_in.go index c9743381b..c2df18bc1 100644 --- a/pkg/sign_in.go +++ b/pkg/sign_in.go @@ -11,6 +11,12 @@ import ( "github.com/kong/go-srp" ) +type accSecretInfo struct { + MasterKey []byte + SecretKey []byte + Token []byte +} + func (c *ClICtrl) signInViaPassword(ctx context.Context, email string, srpAttr *api.SRPAttributes) (*api.AuthorizationResponse, []byte, error) { for { // CLI prompt for password @@ -28,18 +34,18 @@ func (c *ClICtrl) signInViaPassword(ctx context.Context, email string, srpAttr * srpParams := srp.GetParams(4096) identify := []byte(srpAttr.SRPUserID.String()) - salt := utils.Base64DecodeString(srpAttr.SRPSalt) + salt := utils.DecodeBase64(srpAttr.SRPSalt) clientSecret := srp.GenKey() srpClient := srp.NewClient(srpParams, salt, identify, loginKey, clientSecret) clientA := srpClient.ComputeA() - session, err := c.Client.CreateSRPSession(ctx, srpAttr.SRPUserID, utils.BytesToBase64(clientA)) + session, err := c.Client.CreateSRPSession(ctx, srpAttr.SRPUserID, utils.EncodeBase64(clientA)) if err != nil { return nil, nil, err } serverB := session.SRPB - srpClient.SetB(utils.Base64DecodeString(serverB)) + srpClient.SetB(utils.DecodeBase64(serverB)) clientM := srpClient.ComputeM1() - authResp, err := c.Client.VerifySRPSession(ctx, srpAttr.SRPUserID, session.SessionID, utils.BytesToBase64(clientM)) + authResp, err := c.Client.VerifySRPSession(ctx, srpAttr.SRPUserID, session.SessionID, utils.EncodeBase64(clientM)) if err != nil { log.Printf("failed to verify %v", err) continue @@ -55,60 +61,66 @@ func (c *ClICtrl) decryptMasterKeyAndToken( _ context.Context, authResp *api.AuthorizationResponse, keyEncKey []byte, -) (masterKey, token []byte, err error) { +) (*accSecretInfo, error) { var currentKeyEncKey []byte + var masterKey, secretKey, tokenKey []byte + var err error for { if keyEncKey == nil { // CLI prompt for password password, flowErr := GetSensitiveField("Enter password") if flowErr != nil { - return nil, nil, flowErr + return nil, flowErr } fmt.Println("\nPlease wait authenticating...") currentKeyEncKey, err = enteCrypto.DeriveArgonKey(password, authResp.KeyAttributes.KEKSalt, authResp.KeyAttributes.MemLimit, authResp.KeyAttributes.OpsLimit) if err != nil { fmt.Printf("error deriving key encryption key: %v", err) - return nil, nil, err + return nil, err } } else { currentKeyEncKey = keyEncKey } - encryptedKey := utils.Base64DecodeString(authResp.KeyAttributes.EncryptedKey) - encryptedKeyNonce := utils.Base64DecodeString(authResp.KeyAttributes.KeyDecryptionNonce) + encryptedKey := utils.DecodeBase64(authResp.KeyAttributes.EncryptedKey) + encryptedKeyNonce := utils.DecodeBase64(authResp.KeyAttributes.KeyDecryptionNonce) key, keyErr := enteCrypto.SecretBoxOpen(encryptedKey, encryptedKeyNonce, currentKeyEncKey) if keyErr != nil { if keyEncKey != nil { fmt.Printf("Failed to get key from keyEncryptionKey %s", keyErr) - return nil, nil, keyErr + return nil, keyErr } else { fmt.Printf("Incorrect password, error decrypting master key: %v", keyErr) continue } } - masterKey, keyErr = enteCrypto.SecretBoxOpen( - utils.Base64DecodeString(authResp.KeyAttributes.EncryptedSecretKey), - utils.Base64DecodeString(authResp.KeyAttributes.SecretKeyDecryptionNonce), + secretKey, keyErr = enteCrypto.SecretBoxOpen( + utils.DecodeBase64(authResp.KeyAttributes.EncryptedSecretKey), + utils.DecodeBase64(authResp.KeyAttributes.SecretKeyDecryptionNonce), key, ) if keyErr != nil { fmt.Printf("error decrypting master key: %v", keyErr) - return nil, nil, keyErr + return nil, keyErr } - token, err = enteCrypto.SealedBoxOpen( - utils.Base64DecodeString(authResp.EncryptedToken), - utils.Base64DecodeString(authResp.KeyAttributes.PublicKey), - masterKey, + tokenKey, err = enteCrypto.SealedBoxOpen( + utils.DecodeBase64(authResp.EncryptedToken), + utils.DecodeBase64(authResp.KeyAttributes.PublicKey), + secretKey, ) if err != nil { fmt.Printf("error decrypting token: %v", err) - return nil, nil, err + return nil, err } break } - return masterKey, token, nil + return &accSecretInfo{ + MasterKey: masterKey, + SecretKey: secretKey, + Token: tokenKey, + }, nil } func (c *ClICtrl) validateTOTP(ctx context.Context, authResp *api.AuthorizationResponse) (*api.AuthorizationResponse, error) { diff --git a/utils/encoding.go b/utils/encoding.go index a9a9ee6a5..33a5121da 100644 --- a/utils/encoding.go +++ b/utils/encoding.go @@ -4,7 +4,7 @@ import ( "encoding/base64" ) -func Base64DecodeString(s string) []byte { +func DecodeBase64(s string) []byte { b, err := base64.StdEncoding.DecodeString(s) if err != nil { panic(err) @@ -12,6 +12,6 @@ func Base64DecodeString(s string) []byte { return b } -func BytesToBase64(b []byte) string { +func EncodeBase64(b []byte) string { return base64.StdEncoding.EncodeToString(b) } From fcb290a5536ea52dee2c0a02fbc09db7d577af8d Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 22 Sep 2023 19:07:12 +0530 Subject: [PATCH 044/183] Bug fixes and support for decrypting collection names --- internal/api/collection_type.go | 21 +++++++++++++++++++++ internal/crypto/crypto.go | 29 +++++++++++++++++++++++++++++ main.go | 3 ++- pkg/account.go | 7 ++++++- pkg/controller.go | 2 ++ pkg/model/account.go | 4 ++-- pkg/remote_sync.go | 20 +++++++++++++++++--- pkg/sign_in.go | 23 +++++++++++------------ 8 files changed, 90 insertions(+), 19 deletions(-) diff --git a/internal/api/collection_type.go b/internal/api/collection_type.go index 344bffadf..2bf0536bd 100644 --- a/internal/api/collection_type.go +++ b/internal/api/collection_type.go @@ -1,5 +1,11 @@ package api +import ( + enteCrypto "cli-go/internal/crypto" + "cli-go/utils" + "log" +) + // Collection represents a collection type Collection struct { ID int64 `json:"id"` @@ -16,6 +22,21 @@ type Collection struct { MagicMetadata *MagicMetadata `json:"magicMetadata,omitempty"` PublicMagicMetadata *MagicMetadata `json:"pubMagicMetadata,omitempty"` SharedMagicMetadata *MagicMetadata `json:"sharedMagicMetadata,omitempty"` + collectionKey []byte +} + +func (c *Collection) GetCollectionKey(masterKey []byte) []byte { + if c.collectionKey == nil || len(c.collectionKey) == 0 { + collKey, err := enteCrypto.SecretBoxOpen( + utils.DecodeBase64(c.EncryptedKey), + utils.DecodeBase64(c.KeyDecryptionNonce), + masterKey) + if err != nil { + log.Fatalf("failed to decrypt collection key %s", err) + } + c.collectionKey = collKey + } + return c.collectionKey } // CollectionUser represents the owner of a collection diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go index 79111921e..b259dfcaf 100644 --- a/internal/crypto/crypto.go +++ b/internal/crypto/crypto.go @@ -2,6 +2,7 @@ package crypto import ( "bytes" + "cli-go/utils" "encoding/base64" "fmt" "io" @@ -75,6 +76,25 @@ func DecryptChaCha20poly1305(data []byte, key []byte, nonce []byte) ([]byte, err return decryptedData[:n], nil } +func DecryptChaChaBase64(data string, key []byte, nonce string) (string, []byte, error) { + // Decode data from base64 + dataBytes, err := base64.StdEncoding.DecodeString(data) + if err != nil { + return "", nil, fmt.Errorf("invalid data: %v", err) + } + // Decode nonce from base64 + nonceBytes, err := base64.StdEncoding.DecodeString(nonce) + if err != nil { + return "", nil, fmt.Errorf("invalid nonce: %v", err) + } + // Decrypt data + decryptedData, err := DecryptChaCha20poly1305(dataBytes, key, nonceBytes) + if err != nil { + return "", nil, fmt.Errorf("failed to decrypt data: %v", err) + } + return base64.StdEncoding.EncodeToString(decryptedData), decryptedData, nil +} + // EncryptChaCha20poly1305 encrypts the given data using the ChaCha20-Poly1305 algorithm. // Parameters: // - data: The plaintext data as a byte slice. @@ -110,6 +130,15 @@ func SecretBoxOpen(c []byte, n []byte, k []byte) ([]byte, error) { return cp.SecretBoxOpen(sodium.SecretBoxNonce{Bytes: n}, sodium.SecretBoxKey{Bytes: k}) } +func SecretBoxOpenBase64(cipher string, nonce string, k []byte) ([]byte, error) { + var cp sodium.Bytes = utils.DecodeBase64(cipher) + out, err := cp.SecretBoxOpen(sodium.SecretBoxNonce{Bytes: utils.DecodeBase64(nonce)}, sodium.SecretBoxKey{Bytes: k}) + if err != nil { + return nil, err + } + return out, nil +} + func SealedBoxOpen(cipherText []byte, publicKey, masterSecret []byte) ([]byte, error) { var cp sodium.Bytes = cipherText om, err := cp.SealedBoxOpen(sodium.BoxKP{ diff --git a/main.go b/main.go index 2e72e97f0..536337dd1 100644 --- a/main.go +++ b/main.go @@ -16,7 +16,8 @@ func main() { Debug: false, Host: "http://localhost:8080", }), - DB: db, + DB: db, + CliKey: pkg.GetOrCreateClISecret(), } err = ctrl.Init() if err != nil { diff --git a/pkg/account.go b/pkg/account.go index 89d2f435c..3f5c10973 100644 --- a/pkg/account.go +++ b/pkg/account.go @@ -3,6 +3,7 @@ package pkg import ( "cli-go/internal/api" "cli-go/pkg/model" + "cli-go/utils" "context" "encoding/json" "fmt" @@ -65,12 +66,16 @@ func (c *ClICtrl) AddAccount(cxt context.Context) { return } else { fmt.Println("Account added successfully") + // length of master and secret + fmt.Printf("Master key length: %d\n", len(secretInfo.MasterKey)) + fmt.Printf("Secret key length: %d\n", len(secretInfo.SecretKey)) + fmt.Printf("Master key: %s", utils.EncodeBase64(secretInfo.MasterKey)) } } func (c *ClICtrl) storeAccount(_ context.Context, email string, userID int64, app api.App, secretInfo *accSecretInfo) error { // get password - secret := GetOrCreateClISecret() + secret := c.CliKey err := c.DB.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucketIfNotExists([]byte(AccBucket)) if err != nil { diff --git a/pkg/controller.go b/pkg/controller.go index d60e1f0dc..73f3e796c 100644 --- a/pkg/controller.go +++ b/pkg/controller.go @@ -9,6 +9,8 @@ import ( type ClICtrl struct { Client *api.Client DB *bolt.DB + // CliKey is the key used to encrypt/decrypt sensitive data stored in the database + CliKey []byte } func (c *ClICtrl) Init() error { diff --git a/pkg/model/account.go b/pkg/model/account.go index 9031b95fa..779b123a7 100644 --- a/pkg/model/account.go +++ b/pkg/model/account.go @@ -14,10 +14,10 @@ type Account struct { Token EncString `json:"token" binding:"required"` } -func (a Account) AccountKey() string { +func (a *Account) AccountKey() string { return fmt.Sprintf("%s-%d", a.App, a.UserID) } -func (a Account) DataBucket() string { +func (a *Account) DataBucket() string { return fmt.Sprintf("%s-%d-data", a.App, a.UserID) } diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index 52f6741e5..2b21336cf 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -1,6 +1,7 @@ package pkg import ( + enteCrypto "cli-go/internal/crypto" "cli-go/pkg/model" "cli-go/utils" "context" @@ -10,14 +11,17 @@ import ( "log" ) +var accountMasterKey = map[string][]byte{} + func (c *ClICtrl) SyncAccount(account model.Account) error { log.SetPrefix(fmt.Sprintf("[%s] ", account.Email)) - cliSecret := GetOrCreateClISecret() + err := createDataBuckets(c.DB, account) if err != nil { return err } - token := account.Token.MustDecrypt(cliSecret) + token := account.Token.MustDecrypt(c.CliKey) + accountMasterKey[account.AccountKey()] = utils.DecodeBase64(account.MasterKey.MustDecrypt(c.CliKey)) urlEncodedToken := base64.URLEncoding.EncodeToString(utils.DecodeBase64(token)) c.Client.AddToken(account.AccountKey(), urlEncodedToken) ctx := c.GetRequestContext(context.Background(), account) @@ -54,8 +58,18 @@ func (c *ClICtrl) syncRemoteCollections(ctx context.Context, info model.Account) log.Printf("failed to get collections: %s\n", err) return err } + masterKey := accountMasterKey[info.AccountKey()] for _, collection := range collections { - fmt.Printf("Collection %d\n", collection.ID) + if collection.Owner.ID != info.UserID { + fmt.Printf("Skipping collection %d\n", collection.ID) + continue + } + collectionKey := collection.GetCollectionKey(masterKey) + name, nameErr := enteCrypto.SecretBoxOpenBase64(collection.EncryptedName, collection.NameDecryptionNonce, collectionKey) + if nameErr != nil { + log.Fatalf("failed to decrypt collection name: %v", nameErr) + } + fmt.Printf("Collection Name %s\n", string(name)) } return nil } diff --git a/pkg/sign_in.go b/pkg/sign_in.go index c2df18bc1..5c9a8f40c 100644 --- a/pkg/sign_in.go +++ b/pkg/sign_in.go @@ -62,10 +62,9 @@ func (c *ClICtrl) decryptMasterKeyAndToken( authResp *api.AuthorizationResponse, keyEncKey []byte, ) (*accSecretInfo, error) { - var currentKeyEncKey []byte - var masterKey, secretKey, tokenKey []byte var err error + var masterKey, secretKey, tokenKey []byte for { if keyEncKey == nil { // CLI prompt for password @@ -86,24 +85,24 @@ func (c *ClICtrl) decryptMasterKeyAndToken( encryptedKey := utils.DecodeBase64(authResp.KeyAttributes.EncryptedKey) encryptedKeyNonce := utils.DecodeBase64(authResp.KeyAttributes.KeyDecryptionNonce) - key, keyErr := enteCrypto.SecretBoxOpen(encryptedKey, encryptedKeyNonce, currentKeyEncKey) - if keyErr != nil { + masterKey, err = enteCrypto.SecretBoxOpen(encryptedKey, encryptedKeyNonce, currentKeyEncKey) + if err != nil { if keyEncKey != nil { - fmt.Printf("Failed to get key from keyEncryptionKey %s", keyErr) - return nil, keyErr + fmt.Printf("Failed to get key from keyEncryptionKey %s", err) + return nil, err } else { - fmt.Printf("Incorrect password, error decrypting master key: %v", keyErr) + fmt.Printf("Incorrect password, error decrypting master key: %v", err) continue } } - secretKey, keyErr = enteCrypto.SecretBoxOpen( + secretKey, err = enteCrypto.SecretBoxOpen( utils.DecodeBase64(authResp.KeyAttributes.EncryptedSecretKey), utils.DecodeBase64(authResp.KeyAttributes.SecretKeyDecryptionNonce), - key, + masterKey, ) - if keyErr != nil { - fmt.Printf("error decrypting master key: %v", keyErr) - return nil, keyErr + if err != nil { + fmt.Printf("error decrypting master key: %v", err) + return nil, err } tokenKey, err = enteCrypto.SealedBoxOpen( utils.DecodeBase64(authResp.EncryptedToken), From 17aa01ea378aebffef4ac397463b9ed09613fe37 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 22 Sep 2023 19:17:24 +0530 Subject: [PATCH 045/183] Fix test --- main.go | 1 + pkg/key_holder.go | 4 ++++ pkg/model/enc_string.go | 2 +- pkg/remote_sync.go | 9 ++++----- 4 files changed, 10 insertions(+), 6 deletions(-) create mode 100644 pkg/key_holder.go diff --git a/main.go b/main.go index 536337dd1..57433358a 100644 --- a/main.go +++ b/main.go @@ -29,4 +29,5 @@ func main() { } }() cmd.Execute(&ctrl) + //ctrl.StartSync() } diff --git a/pkg/key_holder.go b/pkg/key_holder.go new file mode 100644 index 000000000..284852063 --- /dev/null +++ b/pkg/key_holder.go @@ -0,0 +1,4 @@ +package pkg + +type KeyHolder struct { +} diff --git a/pkg/model/enc_string.go b/pkg/model/enc_string.go index d31c86e12..71604df42 100644 --- a/pkg/model/enc_string.go +++ b/pkg/model/enc_string.go @@ -27,5 +27,5 @@ func (e *EncString) MustDecrypt(key []byte) string { if err != nil { panic(err) } - return utils.EncodeBase64(plainBytes) + return string(plainBytes) } diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index 2b21336cf..21bc33a06 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -3,7 +3,6 @@ package pkg import ( enteCrypto "cli-go/internal/crypto" "cli-go/pkg/model" - "cli-go/utils" "context" "encoding/base64" "fmt" @@ -21,14 +20,14 @@ func (c *ClICtrl) SyncAccount(account model.Account) error { return err } token := account.Token.MustDecrypt(c.CliKey) - accountMasterKey[account.AccountKey()] = utils.DecodeBase64(account.MasterKey.MustDecrypt(c.CliKey)) - urlEncodedToken := base64.URLEncoding.EncodeToString(utils.DecodeBase64(token)) + accountMasterKey[account.AccountKey()] = []byte(account.MasterKey.MustDecrypt(c.CliKey)) + urlEncodedToken := base64.URLEncoding.EncodeToString([]byte(token)) c.Client.AddToken(account.AccountKey(), urlEncodedToken) - ctx := c.GetRequestContext(context.Background(), account) + ctx := c.buildRequestContext(context.Background(), account) return c.syncRemoteCollections(ctx, account) } -func (c *ClICtrl) GetRequestContext(ctx context.Context, account model.Account) context.Context { +func (c *ClICtrl) buildRequestContext(ctx context.Context, account model.Account) context.Context { ctx = context.WithValue(ctx, "app", string(account.App)) ctx = context.WithValue(ctx, "account_id", account.AccountKey()) return ctx From 49966ad7cb8c4bdf43b25d26bf433a52e5526b78 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 22 Sep 2023 21:45:01 +0530 Subject: [PATCH 046/183] Add KeyHolder for keeping in-mem keys --- main.go | 5 +++-- pkg/account.go | 13 ++++--------- pkg/controller.go | 3 ++- pkg/key_holder.go | 23 +++++++++++++++++++++++ pkg/model/enc_string.go | 8 ++++---- pkg/remote_sync.go | 13 +++++++------ pkg/sign_in.go | 2 +- 7 files changed, 44 insertions(+), 23 deletions(-) diff --git a/main.go b/main.go index 57433358a..391e43f80 100644 --- a/main.go +++ b/main.go @@ -16,8 +16,9 @@ func main() { Debug: false, Host: "http://localhost:8080", }), - DB: db, - CliKey: pkg.GetOrCreateClISecret(), + DB: db, + CliKey: pkg.GetOrCreateClISecret(), + KeyHolder: pkg.NewKeyHolder(), } err = ctrl.Init() if err != nil { diff --git a/pkg/account.go b/pkg/account.go index 3f5c10973..c2efabe05 100644 --- a/pkg/account.go +++ b/pkg/account.go @@ -3,7 +3,6 @@ package pkg import ( "cli-go/internal/api" "cli-go/pkg/model" - "cli-go/utils" "context" "encoding/json" "fmt" @@ -55,7 +54,7 @@ func (c *ClICtrl) AddAccount(cxt context.Context) { if authResponse.EncryptedToken == "" || authResponse.KeyAttributes == nil { panic("no encrypted token or keyAttributes") } - secretInfo, decErr := c.decryptMasterKeyAndToken(cxt, authResponse, keyEncKey) + secretInfo, decErr := c.decryptAccSecretInfo(cxt, authResponse, keyEncKey) if decErr != nil { flowErr = decErr return @@ -66,10 +65,6 @@ func (c *ClICtrl) AddAccount(cxt context.Context) { return } else { fmt.Println("Account added successfully") - // length of master and secret - fmt.Printf("Master key length: %d\n", len(secretInfo.MasterKey)) - fmt.Printf("Secret key length: %d\n", len(secretInfo.SecretKey)) - fmt.Printf("Master key: %s", utils.EncodeBase64(secretInfo.MasterKey)) } } @@ -84,9 +79,9 @@ func (c *ClICtrl) storeAccount(_ context.Context, email string, userID int64, ap accInfo := model.Account{ Email: email, UserID: userID, - MasterKey: *model.MakeEncString(string(secretInfo.MasterKey), secret), - Token: *model.MakeEncString(string(secretInfo.Token), secret), - SecretKey: *model.MakeEncString(string(secretInfo.SecretKey), secret), + MasterKey: *model.MakeEncString(secretInfo.MasterKey, secret), + SecretKey: *model.MakeEncString(secretInfo.SecretKey, secret), + Token: *model.MakeEncString(secretInfo.Token, secret), App: app, } accInfoBytes, err := json.Marshal(accInfo) diff --git a/pkg/controller.go b/pkg/controller.go index 73f3e796c..5d794526e 100644 --- a/pkg/controller.go +++ b/pkg/controller.go @@ -10,7 +10,8 @@ type ClICtrl struct { Client *api.Client DB *bolt.DB // CliKey is the key used to encrypt/decrypt sensitive data stored in the database - CliKey []byte + CliKey []byte + KeyHolder *KeyHolder } func (c *ClICtrl) Init() error { diff --git a/pkg/key_holder.go b/pkg/key_holder.go index 284852063..3a880e4da 100644 --- a/pkg/key_holder.go +++ b/pkg/key_holder.go @@ -1,4 +1,27 @@ package pkg +import "cli-go/pkg/model" + type KeyHolder struct { + AccountSecrets map[string]*accSecretInfo + CollectionKeys map[string][]byte +} + +func NewKeyHolder() *KeyHolder { + return &KeyHolder{ + AccountSecrets: make(map[string]*accSecretInfo), + CollectionKeys: make(map[string][]byte), + } +} + +func (k *KeyHolder) LoadSecrets(account model.Account, cliKey []byte) (*accSecretInfo, error) { + tokenKey := account.Token.MustDecrypt(cliKey) + masterKey := account.MasterKey.MustDecrypt(cliKey) + secretKey := account.SecretKey.MustDecrypt(cliKey) + k.AccountSecrets[account.AccountKey()] = &accSecretInfo{ + Token: tokenKey, + MasterKey: masterKey, + SecretKey: secretKey, + } + return k.AccountSecrets[account.AccountKey()], nil } diff --git a/pkg/model/enc_string.go b/pkg/model/enc_string.go index 71604df42..2ea4ee469 100644 --- a/pkg/model/enc_string.go +++ b/pkg/model/enc_string.go @@ -11,8 +11,8 @@ type EncString struct { Nonce string `json:"nonce"` } -func MakeEncString(plainText string, key []byte) *EncString { - cipher, nonce, err := crypto.EncryptChaCha20poly1305([]byte(plainText), key) +func MakeEncString(plainTextBytes []byte, key []byte) *EncString { + cipher, nonce, err := crypto.EncryptChaCha20poly1305(plainTextBytes, key) if err != nil { log.Fatalf("failed to encrypt %s", err) } @@ -22,10 +22,10 @@ func MakeEncString(plainText string, key []byte) *EncString { } } -func (e *EncString) MustDecrypt(key []byte) string { +func (e *EncString) MustDecrypt(key []byte) []byte { plainBytes, err := crypto.DecryptChaCha20poly1305(utils.DecodeBase64(e.CipherText), key, utils.DecodeBase64(e.Nonce)) if err != nil { panic(err) } - return string(plainBytes) + return plainBytes } diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index 21bc33a06..52a222c6c 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -14,15 +14,16 @@ var accountMasterKey = map[string][]byte{} func (c *ClICtrl) SyncAccount(account model.Account) error { log.SetPrefix(fmt.Sprintf("[%s] ", account.Email)) - - err := createDataBuckets(c.DB, account) + secretInfo, err := c.KeyHolder.LoadSecrets(account, c.CliKey) if err != nil { return err } - token := account.Token.MustDecrypt(c.CliKey) - accountMasterKey[account.AccountKey()] = []byte(account.MasterKey.MustDecrypt(c.CliKey)) - urlEncodedToken := base64.URLEncoding.EncodeToString([]byte(token)) - c.Client.AddToken(account.AccountKey(), urlEncodedToken) + err = createDataBuckets(c.DB, account) + if err != nil { + return err + } + accountMasterKey[account.AccountKey()] = secretInfo.MasterKey + c.Client.AddToken(account.AccountKey(), base64.URLEncoding.EncodeToString(secretInfo.Token)) ctx := c.buildRequestContext(context.Background(), account) return c.syncRemoteCollections(ctx, account) } diff --git a/pkg/sign_in.go b/pkg/sign_in.go index 5c9a8f40c..92cb6c75b 100644 --- a/pkg/sign_in.go +++ b/pkg/sign_in.go @@ -57,7 +57,7 @@ func (c *ClICtrl) signInViaPassword(ctx context.Context, email string, srpAttr * // Parameters: // - keyEncKey: key encryption key is derived from user's password. During SRP based login, this key is already derived. // So, we can pass it to avoid asking for password again. -func (c *ClICtrl) decryptMasterKeyAndToken( +func (c *ClICtrl) decryptAccSecretInfo( _ context.Context, authResp *api.AuthorizationResponse, keyEncKey []byte, From c102fe928187ba0571d59efbaa2df68532564755 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 22 Sep 2023 21:45:37 +0530 Subject: [PATCH 047/183] Fix test --- pkg/model/enc_string_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/model/enc_string_test.go b/pkg/model/enc_string_test.go index 2b3866ecc..128a2f941 100644 --- a/pkg/model/enc_string_test.go +++ b/pkg/model/enc_string_test.go @@ -12,9 +12,9 @@ func TestEncString(t *testing.T) { t.Fatalf("error generating key: %v", err) } data := "dataToEncrypt" - encData := MakeEncString(data, key) + encData := MakeEncString([]byte(data), key) decryptedData := encData.MustDecrypt(key) - if decryptedData != data { + if string(decryptedData) != data { t.Fatalf("decrypted data is not equal to original data") } } From 564696e0ef9470284e8eb7cbe16d4607b44f6442 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 23 Sep 2023 09:33:44 +0530 Subject: [PATCH 048/183] Add model for storing albums locally --- pkg/log/debug_print.go | 25 +++++++++++++++++++++++++ pkg/mappers/album_mapper.go | 10 ++++++++++ pkg/model/album.go | 12 ++++++++++++ 3 files changed, 47 insertions(+) create mode 100644 pkg/log/debug_print.go create mode 100644 pkg/mappers/album_mapper.go create mode 100644 pkg/model/album.go diff --git a/pkg/log/debug_print.go b/pkg/log/debug_print.go new file mode 100644 index 000000000..762e504fe --- /dev/null +++ b/pkg/log/debug_print.go @@ -0,0 +1,25 @@ +package log + +import ( + "cli-go/pkg/model" + "fmt" +) + +// This file contains functions that are used to print debug information to the console. + +func PrintAlbum(a *model.Album) { + fmt.Printf("ID: %d\n", a.ID) + fmt.Printf("OwnerID: %d\n", a.OwnerID) + fmt.Printf("AlbumName: %s\n", a.AlbumName) + fmt.Printf("AlbumKey: %s\n", a.AlbumKey.CipherText) + if a.PrivateMeta != nil { + fmt.Printf("PrivateMeta: %s\n", *a.PrivateMeta) + } + if a.PublicMeta != nil { + fmt.Printf("PublicMeta: %s\n", *a.PublicMeta) + } + if a.SharedMeta != nil { + fmt.Printf("SharedMeta: %s\n", *a.SharedMeta) + } + fmt.Printf("LastUpdatedAt: %d\n", a.LastUpdatedAt) +} diff --git a/pkg/mappers/album_mapper.go b/pkg/mappers/album_mapper.go new file mode 100644 index 000000000..bff8365e0 --- /dev/null +++ b/pkg/mappers/album_mapper.go @@ -0,0 +1,10 @@ +package mappers + +import ( + "cli-go/internal/api" + "cli-go/pkg/model" +) + +func MapCollectionToAlbum(collection *api.Collection, collectionKey, cliKey []byte) (*model.Album, error) { + panic("not implemented") +} diff --git a/pkg/model/album.go b/pkg/model/album.go new file mode 100644 index 000000000..0f7272d2b --- /dev/null +++ b/pkg/model/album.go @@ -0,0 +1,12 @@ +package model + +type Album struct { + ID int64 `json:"id"` + OwnerID int64 `json:"ownerID"` + AlbumName string `json:"albumName"` + AlbumKey EncString `json:"albumKey"` + PublicMeta *string `json:"publicMeta"` + PrivateMeta *string `json:"privateMeta"` + SharedMeta *string `json:"sharedMeta"` + LastUpdatedAt int64 `json:"lastUpdatedAt"` +} From 42a6217d44a9b10e105e0aa077c23f0e35042571 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 23 Sep 2023 09:34:21 +0530 Subject: [PATCH 049/183] Refactor computation of collecitonKey --- internal/api/collection_type.go | 20 ------------------- pkg/key_holder.go | 10 +++++++++- pkg/remote_sync.go | 34 +++++++++++++++++++++++++-------- 3 files changed, 35 insertions(+), 29 deletions(-) diff --git a/internal/api/collection_type.go b/internal/api/collection_type.go index 2bf0536bd..f3bba7bb8 100644 --- a/internal/api/collection_type.go +++ b/internal/api/collection_type.go @@ -1,11 +1,5 @@ package api -import ( - enteCrypto "cli-go/internal/crypto" - "cli-go/utils" - "log" -) - // Collection represents a collection type Collection struct { ID int64 `json:"id"` @@ -25,20 +19,6 @@ type Collection struct { collectionKey []byte } -func (c *Collection) GetCollectionKey(masterKey []byte) []byte { - if c.collectionKey == nil || len(c.collectionKey) == 0 { - collKey, err := enteCrypto.SecretBoxOpen( - utils.DecodeBase64(c.EncryptedKey), - utils.DecodeBase64(c.KeyDecryptionNonce), - masterKey) - if err != nil { - log.Fatalf("failed to decrypt collection key %s", err) - } - c.collectionKey = collKey - } - return c.collectionKey -} - // CollectionUser represents the owner of a collection type CollectionUser struct { ID int64 `json:"id"` diff --git a/pkg/key_holder.go b/pkg/key_holder.go index 3a880e4da..939a4ac46 100644 --- a/pkg/key_holder.go +++ b/pkg/key_holder.go @@ -1,6 +1,9 @@ package pkg -import "cli-go/pkg/model" +import ( + "cli-go/pkg/model" + "context" +) type KeyHolder struct { AccountSecrets map[string]*accSecretInfo @@ -25,3 +28,8 @@ func (k *KeyHolder) LoadSecrets(account model.Account, cliKey []byte) (*accSecre } return k.AccountSecrets[account.AccountKey()], nil } + +func (k *KeyHolder) GetAccountSecretInfo(ctx context.Context) *accSecretInfo { + accountKey := ctx.Value("account_id").(string) + return k.AccountSecrets[accountKey] +} diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index 52a222c6c..5576f284e 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -1,8 +1,10 @@ package pkg import ( + "cli-go/internal/api" enteCrypto "cli-go/internal/crypto" "cli-go/pkg/model" + "cli-go/utils" "context" "encoding/base64" "fmt" @@ -10,27 +12,25 @@ import ( "log" ) -var accountMasterKey = map[string][]byte{} - func (c *ClICtrl) SyncAccount(account model.Account) error { log.SetPrefix(fmt.Sprintf("[%s] ", account.Email)) secretInfo, err := c.KeyHolder.LoadSecrets(account, c.CliKey) if err != nil { return err } + ctx := c.buildRequestContext(context.Background(), account) err = createDataBuckets(c.DB, account) if err != nil { return err } - accountMasterKey[account.AccountKey()] = secretInfo.MasterKey c.Client.AddToken(account.AccountKey(), base64.URLEncoding.EncodeToString(secretInfo.Token)) - ctx := c.buildRequestContext(context.Background(), account) return c.syncRemoteCollections(ctx, account) } func (c *ClICtrl) buildRequestContext(ctx context.Context, account model.Account) context.Context { ctx = context.WithValue(ctx, "app", string(account.App)) ctx = context.WithValue(ctx, "account_id", account.AccountKey()) + ctx = context.WithValue(ctx, "user_id", account.UserID) return ctx } @@ -55,16 +55,17 @@ func createDataBuckets(db *bolt.DB, account model.Account) error { func (c *ClICtrl) syncRemoteCollections(ctx context.Context, info model.Account) error { collections, err := c.Client.GetCollections(ctx, 0) if err != nil { - log.Printf("failed to get collections: %s\n", err) - return err + return fmt.Errorf("failed to get collections: %s", err) } - masterKey := accountMasterKey[info.AccountKey()] for _, collection := range collections { if collection.Owner.ID != info.UserID { fmt.Printf("Skipping collection %d\n", collection.ID) continue } - collectionKey := collection.GetCollectionKey(masterKey) + collectionKey, err := c.getCollectionKey(ctx, collection) + if err != nil { + return err + } name, nameErr := enteCrypto.SecretBoxOpenBase64(collection.EncryptedName, collection.NameDecryptionNonce, collectionKey) if nameErr != nil { log.Fatalf("failed to decrypt collection name: %v", nameErr) @@ -73,3 +74,20 @@ func (c *ClICtrl) syncRemoteCollections(ctx context.Context, info model.Account) } return nil } + +func (c *ClICtrl) getCollectionKey(ctx context.Context, collection api.Collection) ([]byte, error) { + accSecretInfo := c.KeyHolder.GetAccountSecretInfo(ctx) + userID := ctx.Value("user_id").(int64) + if collection.Owner.ID == userID { + collKey, err := enteCrypto.SecretBoxOpen( + utils.DecodeBase64(collection.EncryptedKey), + utils.DecodeBase64(collection.KeyDecryptionNonce), + accSecretInfo.MasterKey) + if err != nil { + log.Fatalf("failed to decrypt collection key %s", err) + } + return collKey, nil + } else { + panic("not implemented") + } +} From d59991bd7ff550617c5ae7e872c81f6d815f2ed6 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 23 Sep 2023 09:35:37 +0530 Subject: [PATCH 050/183] Move encoding utils in separate pkg --- internal/crypto/crypto.go | 6 +++--- pkg/model/enc_string.go | 8 ++++---- pkg/remote_sync.go | 6 +++--- pkg/sign_in.go | 22 +++++++++++----------- utils/{ => encoding}/encoding.go | 2 +- 5 files changed, 22 insertions(+), 22 deletions(-) rename utils/{ => encoding}/encoding.go (93%) diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go index b259dfcaf..c66409b97 100644 --- a/internal/crypto/crypto.go +++ b/internal/crypto/crypto.go @@ -2,7 +2,7 @@ package crypto import ( "bytes" - "cli-go/utils" + "cli-go/utils/encoding" "encoding/base64" "fmt" "io" @@ -131,8 +131,8 @@ func SecretBoxOpen(c []byte, n []byte, k []byte) ([]byte, error) { } func SecretBoxOpenBase64(cipher string, nonce string, k []byte) ([]byte, error) { - var cp sodium.Bytes = utils.DecodeBase64(cipher) - out, err := cp.SecretBoxOpen(sodium.SecretBoxNonce{Bytes: utils.DecodeBase64(nonce)}, sodium.SecretBoxKey{Bytes: k}) + var cp sodium.Bytes = encoding.DecodeBase64(cipher) + out, err := cp.SecretBoxOpen(sodium.SecretBoxNonce{Bytes: encoding.DecodeBase64(nonce)}, sodium.SecretBoxKey{Bytes: k}) if err != nil { return nil, err } diff --git a/pkg/model/enc_string.go b/pkg/model/enc_string.go index 2ea4ee469..09f480860 100644 --- a/pkg/model/enc_string.go +++ b/pkg/model/enc_string.go @@ -2,7 +2,7 @@ package model import ( "cli-go/internal/crypto" - "cli-go/utils" + "cli-go/utils/encoding" "log" ) @@ -17,13 +17,13 @@ func MakeEncString(plainTextBytes []byte, key []byte) *EncString { log.Fatalf("failed to encrypt %s", err) } return &EncString{ - CipherText: utils.EncodeBase64(cipher), - Nonce: utils.EncodeBase64(nonce), + CipherText: encoding.EncodeBase64(cipher), + Nonce: encoding.EncodeBase64(nonce), } } func (e *EncString) MustDecrypt(key []byte) []byte { - plainBytes, err := crypto.DecryptChaCha20poly1305(utils.DecodeBase64(e.CipherText), key, utils.DecodeBase64(e.Nonce)) + plainBytes, err := crypto.DecryptChaCha20poly1305(encoding.DecodeBase64(e.CipherText), key, encoding.DecodeBase64(e.Nonce)) if err != nil { panic(err) } diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index 5576f284e..d2a24f144 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -4,7 +4,7 @@ import ( "cli-go/internal/api" enteCrypto "cli-go/internal/crypto" "cli-go/pkg/model" - "cli-go/utils" + "cli-go/utils/encoding" "context" "encoding/base64" "fmt" @@ -80,8 +80,8 @@ func (c *ClICtrl) getCollectionKey(ctx context.Context, collection api.Collectio userID := ctx.Value("user_id").(int64) if collection.Owner.ID == userID { collKey, err := enteCrypto.SecretBoxOpen( - utils.DecodeBase64(collection.EncryptedKey), - utils.DecodeBase64(collection.KeyDecryptionNonce), + encoding.DecodeBase64(collection.EncryptedKey), + encoding.DecodeBase64(collection.KeyDecryptionNonce), accSecretInfo.MasterKey) if err != nil { log.Fatalf("failed to decrypt collection key %s", err) diff --git a/pkg/sign_in.go b/pkg/sign_in.go index 92cb6c75b..180183fe2 100644 --- a/pkg/sign_in.go +++ b/pkg/sign_in.go @@ -3,7 +3,7 @@ package pkg import ( "cli-go/internal/api" enteCrypto "cli-go/internal/crypto" - "cli-go/utils" + "cli-go/utils/encoding" "context" "fmt" "log" @@ -34,18 +34,18 @@ func (c *ClICtrl) signInViaPassword(ctx context.Context, email string, srpAttr * srpParams := srp.GetParams(4096) identify := []byte(srpAttr.SRPUserID.String()) - salt := utils.DecodeBase64(srpAttr.SRPSalt) + salt := encoding.DecodeBase64(srpAttr.SRPSalt) clientSecret := srp.GenKey() srpClient := srp.NewClient(srpParams, salt, identify, loginKey, clientSecret) clientA := srpClient.ComputeA() - session, err := c.Client.CreateSRPSession(ctx, srpAttr.SRPUserID, utils.EncodeBase64(clientA)) + session, err := c.Client.CreateSRPSession(ctx, srpAttr.SRPUserID, encoding.EncodeBase64(clientA)) if err != nil { return nil, nil, err } serverB := session.SRPB - srpClient.SetB(utils.DecodeBase64(serverB)) + srpClient.SetB(encoding.DecodeBase64(serverB)) clientM := srpClient.ComputeM1() - authResp, err := c.Client.VerifySRPSession(ctx, srpAttr.SRPUserID, session.SessionID, utils.EncodeBase64(clientM)) + authResp, err := c.Client.VerifySRPSession(ctx, srpAttr.SRPUserID, session.SessionID, encoding.EncodeBase64(clientM)) if err != nil { log.Printf("failed to verify %v", err) continue @@ -83,8 +83,8 @@ func (c *ClICtrl) decryptAccSecretInfo( currentKeyEncKey = keyEncKey } - encryptedKey := utils.DecodeBase64(authResp.KeyAttributes.EncryptedKey) - encryptedKeyNonce := utils.DecodeBase64(authResp.KeyAttributes.KeyDecryptionNonce) + encryptedKey := encoding.DecodeBase64(authResp.KeyAttributes.EncryptedKey) + encryptedKeyNonce := encoding.DecodeBase64(authResp.KeyAttributes.KeyDecryptionNonce) masterKey, err = enteCrypto.SecretBoxOpen(encryptedKey, encryptedKeyNonce, currentKeyEncKey) if err != nil { if keyEncKey != nil { @@ -96,8 +96,8 @@ func (c *ClICtrl) decryptAccSecretInfo( } } secretKey, err = enteCrypto.SecretBoxOpen( - utils.DecodeBase64(authResp.KeyAttributes.EncryptedSecretKey), - utils.DecodeBase64(authResp.KeyAttributes.SecretKeyDecryptionNonce), + encoding.DecodeBase64(authResp.KeyAttributes.EncryptedSecretKey), + encoding.DecodeBase64(authResp.KeyAttributes.SecretKeyDecryptionNonce), masterKey, ) if err != nil { @@ -105,8 +105,8 @@ func (c *ClICtrl) decryptAccSecretInfo( return nil, err } tokenKey, err = enteCrypto.SealedBoxOpen( - utils.DecodeBase64(authResp.EncryptedToken), - utils.DecodeBase64(authResp.KeyAttributes.PublicKey), + encoding.DecodeBase64(authResp.EncryptedToken), + encoding.DecodeBase64(authResp.KeyAttributes.PublicKey), secretKey, ) if err != nil { diff --git a/utils/encoding.go b/utils/encoding/encoding.go similarity index 93% rename from utils/encoding.go rename to utils/encoding/encoding.go index 33a5121da..b5cec0b5f 100644 --- a/utils/encoding.go +++ b/utils/encoding/encoding.go @@ -1,4 +1,4 @@ -package utils +package encoding import ( "encoding/base64" From 02b08da7eab1ae6f1f7a9d39ef4f85d092355d0f Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 23 Sep 2023 09:40:54 +0530 Subject: [PATCH 051/183] Store account publicKey --- pkg/account.go | 2 ++ pkg/key_holder.go | 2 ++ pkg/model/account.go | 2 ++ pkg/sign_in.go | 5 ++++- 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/account.go b/pkg/account.go index c2efabe05..692a08b70 100644 --- a/pkg/account.go +++ b/pkg/account.go @@ -3,6 +3,7 @@ package pkg import ( "cli-go/internal/api" "cli-go/pkg/model" + "cli-go/utils/encoding" "context" "encoding/json" "fmt" @@ -83,6 +84,7 @@ func (c *ClICtrl) storeAccount(_ context.Context, email string, userID int64, ap SecretKey: *model.MakeEncString(secretInfo.SecretKey, secret), Token: *model.MakeEncString(secretInfo.Token, secret), App: app, + PublicKey: encoding.EncodeBase64(secretInfo.PublicKey), } accInfoBytes, err := json.Marshal(accInfo) if err != nil { diff --git a/pkg/key_holder.go b/pkg/key_holder.go index 939a4ac46..f3e266499 100644 --- a/pkg/key_holder.go +++ b/pkg/key_holder.go @@ -2,6 +2,7 @@ package pkg import ( "cli-go/pkg/model" + "cli-go/utils/encoding" "context" ) @@ -25,6 +26,7 @@ func (k *KeyHolder) LoadSecrets(account model.Account, cliKey []byte) (*accSecre Token: tokenKey, MasterKey: masterKey, SecretKey: secretKey, + PublicKey: encoding.DecodeBase64(account.PublicKey), } return k.AccountSecrets[account.AccountKey()], nil } diff --git a/pkg/model/account.go b/pkg/model/account.go index 779b123a7..8556ec8ab 100644 --- a/pkg/model/account.go +++ b/pkg/model/account.go @@ -11,6 +11,8 @@ type Account struct { App api.App `json:"app" binding:"required"` MasterKey EncString `json:"masterKey" binding:"required"` SecretKey EncString `json:"secretKey" binding:"required"` + // PublicKey corresponding to the secret key + PublicKey string `json:"publicKey" binding:"required"` Token EncString `json:"token" binding:"required"` } diff --git a/pkg/sign_in.go b/pkg/sign_in.go index 180183fe2..44abdbde4 100644 --- a/pkg/sign_in.go +++ b/pkg/sign_in.go @@ -15,6 +15,7 @@ type accSecretInfo struct { MasterKey []byte SecretKey []byte Token []byte + PublicKey []byte } func (c *ClICtrl) signInViaPassword(ctx context.Context, email string, srpAttr *api.SRPAttributes) (*api.AuthorizationResponse, []byte, error) { @@ -65,6 +66,7 @@ func (c *ClICtrl) decryptAccSecretInfo( var currentKeyEncKey []byte var err error var masterKey, secretKey, tokenKey []byte + var publicKey = encoding.DecodeBase64(authResp.KeyAttributes.PublicKey) for { if keyEncKey == nil { // CLI prompt for password @@ -106,7 +108,7 @@ func (c *ClICtrl) decryptAccSecretInfo( } tokenKey, err = enteCrypto.SealedBoxOpen( encoding.DecodeBase64(authResp.EncryptedToken), - encoding.DecodeBase64(authResp.KeyAttributes.PublicKey), + publicKey, secretKey, ) if err != nil { @@ -119,6 +121,7 @@ func (c *ClICtrl) decryptAccSecretInfo( MasterKey: masterKey, SecretKey: secretKey, Token: tokenKey, + PublicKey: publicKey, }, nil } From c5f978d32bb8804a4783e4579cdc5f528424ca91 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 23 Sep 2023 09:47:08 +0530 Subject: [PATCH 052/183] Add support for decrypting shared collectionKey --- pkg/remote_sync.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index d2a24f144..590155f61 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -58,10 +58,7 @@ func (c *ClICtrl) syncRemoteCollections(ctx context.Context, info model.Account) return fmt.Errorf("failed to get collections: %s", err) } for _, collection := range collections { - if collection.Owner.ID != info.UserID { - fmt.Printf("Skipping collection %d\n", collection.ID) - continue - } + collectionKey, err := c.getCollectionKey(ctx, collection) if err != nil { return err @@ -70,7 +67,12 @@ func (c *ClICtrl) syncRemoteCollections(ctx context.Context, info model.Account) if nameErr != nil { log.Fatalf("failed to decrypt collection name: %v", nameErr) } - fmt.Printf("Collection Name %s\n", string(name)) + if collection.Owner.ID != info.UserID { + fmt.Printf("Shared Album %s\n", string(name)) + continue + } else { + fmt.Printf("Owned Name %s\n", string(name)) + } } return nil } @@ -88,6 +90,11 @@ func (c *ClICtrl) getCollectionKey(ctx context.Context, collection api.Collectio } return collKey, nil } else { - panic("not implemented") + collKey, err := enteCrypto.SealedBoxOpen(encoding.DecodeBase64(collection.EncryptedKey), + accSecretInfo.PublicKey, accSecretInfo.SecretKey) + if err != nil { + log.Fatalf("failed to decrypt collection key %s", err) + } + return collKey, nil } } From 02ff452c09a329b943c6217811a03bdf8aa20cc2 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 23 Sep 2023 16:15:10 +0530 Subject: [PATCH 053/183] Refactor --- {pkg => internal}/promt.go | 2 +- main.go | 5 +-- pkg/account.go | 7 ++-- pkg/collections.go | 57 +++++++++++++++++++++++++++++++++ pkg/controller.go | 3 +- pkg/model/account.go | 7 ++++ pkg/remote_sync.go | 50 ----------------------------- pkg/{ => secrets}/key_holder.go | 12 +++---- pkg/{ => secrets}/secret.go | 2 +- pkg/sign_in.go | 21 +++++------- 10 files changed, 89 insertions(+), 77 deletions(-) rename {pkg => internal}/promt.go (98%) create mode 100644 pkg/collections.go rename pkg/{ => secrets}/key_holder.go (69%) rename pkg/{ => secrets}/secret.go (97%) diff --git a/pkg/promt.go b/internal/promt.go similarity index 98% rename from pkg/promt.go rename to internal/promt.go index 65abb0b09..5a9145333 100644 --- a/pkg/promt.go +++ b/internal/promt.go @@ -1,4 +1,4 @@ -package pkg +package internal import ( "cli-go/internal/api" diff --git a/main.go b/main.go index 391e43f80..c5831e766 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,7 @@ import ( "cli-go/cmd" "cli-go/internal/api" "cli-go/pkg" + "cli-go/pkg/secrets" ) func main() { @@ -17,8 +18,8 @@ func main() { Host: "http://localhost:8080", }), DB: db, - CliKey: pkg.GetOrCreateClISecret(), - KeyHolder: pkg.NewKeyHolder(), + CliKey: secrets.GetOrCreateClISecret(), + KeyHolder: secrets.NewKeyHolder(), } err = ctrl.Init() if err != nil { diff --git a/pkg/account.go b/pkg/account.go index 692a08b70..87e145636 100644 --- a/pkg/account.go +++ b/pkg/account.go @@ -1,6 +1,7 @@ package pkg import ( + "cli-go/internal" "cli-go/internal/api" "cli-go/pkg/model" "cli-go/utils/encoding" @@ -21,9 +22,9 @@ func (c *ClICtrl) AddAccount(cxt context.Context) { log.Fatal(flowErr) } }() - app := GetAppType() + app := internal.GetAppType() cxt = context.WithValue(cxt, "app", string(app)) - email, flowErr := GetUserInput("Enter email address") + email, flowErr := internal.GetUserInput("Enter email address") if flowErr != nil { return } @@ -69,7 +70,7 @@ func (c *ClICtrl) AddAccount(cxt context.Context) { } } -func (c *ClICtrl) storeAccount(_ context.Context, email string, userID int64, app api.App, secretInfo *accSecretInfo) error { +func (c *ClICtrl) storeAccount(_ context.Context, email string, userID int64, app api.App, secretInfo *model.AccSecretInfo) error { // get password secret := c.CliKey err := c.DB.Update(func(tx *bolt.Tx) error { diff --git a/pkg/collections.go b/pkg/collections.go new file mode 100644 index 000000000..93886a7ae --- /dev/null +++ b/pkg/collections.go @@ -0,0 +1,57 @@ +package pkg + +import ( + "cli-go/internal/api" + enteCrypto "cli-go/internal/crypto" + "cli-go/pkg/model" + "cli-go/utils/encoding" + "context" + "fmt" + "log" +) + +func (c *ClICtrl) syncRemoteCollections(ctx context.Context, info model.Account) error { + collections, err := c.Client.GetCollections(ctx, 0) + if err != nil { + return fmt.Errorf("failed to get collections: %s", err) + } + for _, collection := range collections { + collectionKey, err := c.getCollectionKey(ctx, collection) + if err != nil { + return err + } + name, nameErr := enteCrypto.SecretBoxOpenBase64(collection.EncryptedName, collection.NameDecryptionNonce, collectionKey) + if nameErr != nil { + log.Fatalf("failed to decrypt collection name: %v", nameErr) + } + if collection.Owner.ID != info.UserID { + fmt.Printf("Shared Album %s\n", string(name)) + continue + } else { + fmt.Printf("Owned Name %s\n", string(name)) + } + } + return nil +} + +func (c *ClICtrl) getCollectionKey(ctx context.Context, collection api.Collection) ([]byte, error) { + accSecretInfo := c.KeyHolder.GetAccountSecretInfo(ctx) + userID := ctx.Value("user_id").(int64) + if collection.Owner.ID == userID { + collKey, err := enteCrypto.SecretBoxOpen( + encoding.DecodeBase64(collection.EncryptedKey), + encoding.DecodeBase64(collection.KeyDecryptionNonce), + accSecretInfo.MasterKey) + if err != nil { + log.Fatalf("failed to decrypt collection key %s", err) + } + return collKey, nil + } else { + collKey, err := enteCrypto.SealedBoxOpen(encoding.DecodeBase64(collection.EncryptedKey), + accSecretInfo.PublicKey, accSecretInfo.SecretKey) + if err != nil { + log.Fatalf("failed to decrypt collection key %s", err) + } + return collKey, nil + } +} diff --git a/pkg/controller.go b/pkg/controller.go index 5d794526e..3d2dd2ed0 100644 --- a/pkg/controller.go +++ b/pkg/controller.go @@ -2,6 +2,7 @@ package pkg import ( "cli-go/internal/api" + "cli-go/pkg/secrets" "fmt" bolt "go.etcd.io/bbolt" ) @@ -11,7 +12,7 @@ type ClICtrl struct { DB *bolt.DB // CliKey is the key used to encrypt/decrypt sensitive data stored in the database CliKey []byte - KeyHolder *KeyHolder + KeyHolder *secrets.KeyHolder } func (c *ClICtrl) Init() error { diff --git a/pkg/model/account.go b/pkg/model/account.go index 8556ec8ab..f721835cb 100644 --- a/pkg/model/account.go +++ b/pkg/model/account.go @@ -23,3 +23,10 @@ func (a *Account) AccountKey() string { func (a *Account) DataBucket() string { return fmt.Sprintf("%s-%d-data", a.App, a.UserID) } + +type AccSecretInfo struct { + MasterKey []byte + SecretKey []byte + Token []byte + PublicKey []byte +} diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index 590155f61..9187fb55d 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -1,10 +1,7 @@ package pkg import ( - "cli-go/internal/api" - enteCrypto "cli-go/internal/crypto" "cli-go/pkg/model" - "cli-go/utils/encoding" "context" "encoding/base64" "fmt" @@ -51,50 +48,3 @@ func createDataBuckets(db *bolt.DB, account model.Account) error { return nil }) } - -func (c *ClICtrl) syncRemoteCollections(ctx context.Context, info model.Account) error { - collections, err := c.Client.GetCollections(ctx, 0) - if err != nil { - return fmt.Errorf("failed to get collections: %s", err) - } - for _, collection := range collections { - - collectionKey, err := c.getCollectionKey(ctx, collection) - if err != nil { - return err - } - name, nameErr := enteCrypto.SecretBoxOpenBase64(collection.EncryptedName, collection.NameDecryptionNonce, collectionKey) - if nameErr != nil { - log.Fatalf("failed to decrypt collection name: %v", nameErr) - } - if collection.Owner.ID != info.UserID { - fmt.Printf("Shared Album %s\n", string(name)) - continue - } else { - fmt.Printf("Owned Name %s\n", string(name)) - } - } - return nil -} - -func (c *ClICtrl) getCollectionKey(ctx context.Context, collection api.Collection) ([]byte, error) { - accSecretInfo := c.KeyHolder.GetAccountSecretInfo(ctx) - userID := ctx.Value("user_id").(int64) - if collection.Owner.ID == userID { - collKey, err := enteCrypto.SecretBoxOpen( - encoding.DecodeBase64(collection.EncryptedKey), - encoding.DecodeBase64(collection.KeyDecryptionNonce), - accSecretInfo.MasterKey) - if err != nil { - log.Fatalf("failed to decrypt collection key %s", err) - } - return collKey, nil - } else { - collKey, err := enteCrypto.SealedBoxOpen(encoding.DecodeBase64(collection.EncryptedKey), - accSecretInfo.PublicKey, accSecretInfo.SecretKey) - if err != nil { - log.Fatalf("failed to decrypt collection key %s", err) - } - return collKey, nil - } -} diff --git a/pkg/key_holder.go b/pkg/secrets/key_holder.go similarity index 69% rename from pkg/key_holder.go rename to pkg/secrets/key_holder.go index f3e266499..198d6a2e4 100644 --- a/pkg/key_holder.go +++ b/pkg/secrets/key_holder.go @@ -1,4 +1,4 @@ -package pkg +package secrets import ( "cli-go/pkg/model" @@ -7,22 +7,22 @@ import ( ) type KeyHolder struct { - AccountSecrets map[string]*accSecretInfo + AccountSecrets map[string]*model.AccSecretInfo CollectionKeys map[string][]byte } func NewKeyHolder() *KeyHolder { return &KeyHolder{ - AccountSecrets: make(map[string]*accSecretInfo), + AccountSecrets: make(map[string]*model.AccSecretInfo), CollectionKeys: make(map[string][]byte), } } -func (k *KeyHolder) LoadSecrets(account model.Account, cliKey []byte) (*accSecretInfo, error) { +func (k *KeyHolder) LoadSecrets(account model.Account, cliKey []byte) (*model.AccSecretInfo, error) { tokenKey := account.Token.MustDecrypt(cliKey) masterKey := account.MasterKey.MustDecrypt(cliKey) secretKey := account.SecretKey.MustDecrypt(cliKey) - k.AccountSecrets[account.AccountKey()] = &accSecretInfo{ + k.AccountSecrets[account.AccountKey()] = &model.AccSecretInfo{ Token: tokenKey, MasterKey: masterKey, SecretKey: secretKey, @@ -31,7 +31,7 @@ func (k *KeyHolder) LoadSecrets(account model.Account, cliKey []byte) (*accSecre return k.AccountSecrets[account.AccountKey()], nil } -func (k *KeyHolder) GetAccountSecretInfo(ctx context.Context) *accSecretInfo { +func (k *KeyHolder) GetAccountSecretInfo(ctx context.Context) *model.AccSecretInfo { accountKey := ctx.Value("account_id").(string) return k.AccountSecrets[accountKey] } diff --git a/pkg/secret.go b/pkg/secrets/secret.go similarity index 97% rename from pkg/secret.go rename to pkg/secrets/secret.go index 3d440863d..3c67bc7b0 100644 --- a/pkg/secret.go +++ b/pkg/secrets/secret.go @@ -1,4 +1,4 @@ -package pkg +package secrets import ( "crypto/rand" diff --git a/pkg/sign_in.go b/pkg/sign_in.go index 44abdbde4..f1e0b970f 100644 --- a/pkg/sign_in.go +++ b/pkg/sign_in.go @@ -1,8 +1,10 @@ package pkg import ( + "cli-go/internal" "cli-go/internal/api" enteCrypto "cli-go/internal/crypto" + "cli-go/pkg/model" "cli-go/utils/encoding" "context" "fmt" @@ -11,17 +13,10 @@ import ( "github.com/kong/go-srp" ) -type accSecretInfo struct { - MasterKey []byte - SecretKey []byte - Token []byte - PublicKey []byte -} - func (c *ClICtrl) signInViaPassword(ctx context.Context, email string, srpAttr *api.SRPAttributes) (*api.AuthorizationResponse, []byte, error) { for { // CLI prompt for password - password, flowErr := GetSensitiveField("Enter password") + password, flowErr := internal.GetSensitiveField("Enter password") if flowErr != nil { return nil, nil, flowErr } @@ -62,7 +57,7 @@ func (c *ClICtrl) decryptAccSecretInfo( _ context.Context, authResp *api.AuthorizationResponse, keyEncKey []byte, -) (*accSecretInfo, error) { +) (*model.AccSecretInfo, error) { var currentKeyEncKey []byte var err error var masterKey, secretKey, tokenKey []byte @@ -70,7 +65,7 @@ func (c *ClICtrl) decryptAccSecretInfo( for { if keyEncKey == nil { // CLI prompt for password - password, flowErr := GetSensitiveField("Enter password") + password, flowErr := internal.GetSensitiveField("Enter password") if flowErr != nil { return nil, flowErr } @@ -117,7 +112,7 @@ func (c *ClICtrl) decryptAccSecretInfo( } break } - return &accSecretInfo{ + return &model.AccSecretInfo{ MasterKey: masterKey, SecretKey: secretKey, Token: tokenKey, @@ -131,7 +126,7 @@ func (c *ClICtrl) validateTOTP(ctx context.Context, authResp *api.AuthorizationR } for { // CLI prompt for TOTP - totp, flowErr := GetCode("Enter TOTP", 6) + totp, flowErr := internal.GetCode("Enter TOTP", 6) if flowErr != nil { return nil, flowErr } @@ -151,7 +146,7 @@ func (c *ClICtrl) validateEmail(ctx context.Context, email string) (*api.Authori } for { // CLI prompt for OTP - ott, flowErr := GetCode("Enter OTP", 6) + ott, flowErr := internal.GetCode("Enter OTP", 6) if flowErr != nil { return nil, flowErr } From 8f6b544f490e2ed7ba2cbe4fdefbec35c15a46f1 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 23 Sep 2023 17:00:37 +0530 Subject: [PATCH 054/183] Move collection key derivation in KeyHolder --- pkg/collections.go | 24 +----------------------- pkg/secrets/key_holder.go | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/pkg/collections.go b/pkg/collections.go index 93886a7ae..b1149c1ec 100644 --- a/pkg/collections.go +++ b/pkg/collections.go @@ -16,7 +16,7 @@ func (c *ClICtrl) syncRemoteCollections(ctx context.Context, info model.Account) return fmt.Errorf("failed to get collections: %s", err) } for _, collection := range collections { - collectionKey, err := c.getCollectionKey(ctx, collection) + collectionKey, err := c.KeyHolder.GetCollectionKey(ctx, collection) if err != nil { return err } @@ -33,25 +33,3 @@ func (c *ClICtrl) syncRemoteCollections(ctx context.Context, info model.Account) } return nil } - -func (c *ClICtrl) getCollectionKey(ctx context.Context, collection api.Collection) ([]byte, error) { - accSecretInfo := c.KeyHolder.GetAccountSecretInfo(ctx) - userID := ctx.Value("user_id").(int64) - if collection.Owner.ID == userID { - collKey, err := enteCrypto.SecretBoxOpen( - encoding.DecodeBase64(collection.EncryptedKey), - encoding.DecodeBase64(collection.KeyDecryptionNonce), - accSecretInfo.MasterKey) - if err != nil { - log.Fatalf("failed to decrypt collection key %s", err) - } - return collKey, nil - } else { - collKey, err := enteCrypto.SealedBoxOpen(encoding.DecodeBase64(collection.EncryptedKey), - accSecretInfo.PublicKey, accSecretInfo.SecretKey) - if err != nil { - log.Fatalf("failed to decrypt collection key %s", err) - } - return collKey, nil - } -} diff --git a/pkg/secrets/key_holder.go b/pkg/secrets/key_holder.go index 198d6a2e4..d224f7469 100644 --- a/pkg/secrets/key_holder.go +++ b/pkg/secrets/key_holder.go @@ -1,9 +1,12 @@ package secrets import ( + "cli-go/internal/api" + enteCrypto "cli-go/internal/crypto" "cli-go/pkg/model" "cli-go/utils/encoding" "context" + "fmt" ) type KeyHolder struct { @@ -35,3 +38,25 @@ func (k *KeyHolder) GetAccountSecretInfo(ctx context.Context) *model.AccSecretIn accountKey := ctx.Value("account_id").(string) return k.AccountSecrets[accountKey] } + +func (k *KeyHolder) GetCollectionKey(ctx context.Context, collection api.Collection) ([]byte, error) { + accSecretInfo := k.GetAccountSecretInfo(ctx) + userID := ctx.Value("user_id").(int64) + if collection.Owner.ID == userID { + collKey, err := enteCrypto.SecretBoxOpen( + encoding.DecodeBase64(collection.EncryptedKey), + encoding.DecodeBase64(collection.KeyDecryptionNonce), + accSecretInfo.MasterKey) + if err != nil { + return nil, fmt.Errorf("collection %d key drive failed %s", collection.ID, err) + } + return collKey, nil + } else { + collKey, err := enteCrypto.SealedBoxOpen(encoding.DecodeBase64(collection.EncryptedKey), + accSecretInfo.PublicKey, accSecretInfo.SecretKey) + if err != nil { + return nil, fmt.Errorf("shared collection %d key drive failed %s", collection.ID, err) + } + return collKey, nil + } +} From 66f20598c2a1611c430f19d5529ec2d3af56ea82 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sun, 24 Sep 2023 10:33:59 +0530 Subject: [PATCH 055/183] Documentation --- pkg/secrets/key_holder.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/secrets/key_holder.go b/pkg/secrets/key_holder.go index d224f7469..3ceb5c779 100644 --- a/pkg/secrets/key_holder.go +++ b/pkg/secrets/key_holder.go @@ -21,6 +21,10 @@ func NewKeyHolder() *KeyHolder { } } +// LoadSecrets loads the secrets for a given account using the provided CLI key. +// It decrypts the token key, master key, and secret key using the CLI key. +// The decrypted keys and the decoded public key are stored in the AccountSecrets map using the account key as the map key. +// It returns the account secret information or an error if the decryption fails. func (k *KeyHolder) LoadSecrets(account model.Account, cliKey []byte) (*model.AccSecretInfo, error) { tokenKey := account.Token.MustDecrypt(cliKey) masterKey := account.MasterKey.MustDecrypt(cliKey) @@ -39,6 +43,11 @@ func (k *KeyHolder) GetAccountSecretInfo(ctx context.Context) *model.AccSecretIn return k.AccountSecrets[accountKey] } +// GetCollectionKey retrieves the key for a given collection. +// It first fetches the account secret information from the context. +// If the collection owner's ID matches the user ID from the context, it decrypts the collection key using the master key. +// If the collection is shared (i.e., the owner's ID does not match the user ID), it decrypts the collection key using the public and secret keys. +// It returns the decrypted collection key or an error if the decryption fails. func (k *KeyHolder) GetCollectionKey(ctx context.Context, collection api.Collection) ([]byte, error) { accSecretInfo := k.GetAccountSecretInfo(ctx) userID := ctx.Value("user_id").(int64) From b6417d09b31ce0a255fc5e626a782b7dca687572 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sun, 24 Sep 2023 11:04:51 +0530 Subject: [PATCH 056/183] Mapper for api.Collection to model.Album conversion --- pkg/collections.go | 23 ++++------------ pkg/log/debug_print.go | 12 +++++--- pkg/mappers.go | 55 +++++++++++++++++++++++++++++++++++++ pkg/mappers/album_mapper.go | 10 ------- pkg/model/album.go | 2 ++ 5 files changed, 71 insertions(+), 31 deletions(-) create mode 100644 pkg/mappers.go delete mode 100644 pkg/mappers/album_mapper.go diff --git a/pkg/collections.go b/pkg/collections.go index b1149c1ec..72e26eeb8 100644 --- a/pkg/collections.go +++ b/pkg/collections.go @@ -1,13 +1,10 @@ package pkg import ( - "cli-go/internal/api" - enteCrypto "cli-go/internal/crypto" + debuglog "cli-go/pkg/log" "cli-go/pkg/model" - "cli-go/utils/encoding" "context" "fmt" - "log" ) func (c *ClICtrl) syncRemoteCollections(ctx context.Context, info model.Account) error { @@ -15,21 +12,13 @@ func (c *ClICtrl) syncRemoteCollections(ctx context.Context, info model.Account) if err != nil { return fmt.Errorf("failed to get collections: %s", err) } + for _, collection := range collections { - collectionKey, err := c.KeyHolder.GetCollectionKey(ctx, collection) - if err != nil { - return err - } - name, nameErr := enteCrypto.SecretBoxOpenBase64(collection.EncryptedName, collection.NameDecryptionNonce, collectionKey) - if nameErr != nil { - log.Fatalf("failed to decrypt collection name: %v", nameErr) - } - if collection.Owner.ID != info.UserID { - fmt.Printf("Shared Album %s\n", string(name)) - continue - } else { - fmt.Printf("Owned Name %s\n", string(name)) + album, err2 := c.mapCollectionToAlbum(ctx, collection) + if err2 != nil { + return err2 } + debuglog.PrintAlbum(album) } return nil } diff --git a/pkg/log/debug_print.go b/pkg/log/debug_print.go index 762e504fe..35ac29c38 100644 --- a/pkg/log/debug_print.go +++ b/pkg/log/debug_print.go @@ -1,4 +1,4 @@ -package log +package debuglog import ( "cli-go/pkg/model" @@ -8,10 +8,13 @@ import ( // This file contains functions that are used to print debug information to the console. func PrintAlbum(a *model.Album) { + fmt.Printf("=======\n") fmt.Printf("ID: %d\n", a.ID) fmt.Printf("OwnerID: %d\n", a.OwnerID) - fmt.Printf("AlbumName: %s\n", a.AlbumName) - fmt.Printf("AlbumKey: %s\n", a.AlbumKey.CipherText) + if a.IsShared { + fmt.Printf("Shared album") + } + fmt.Printf(" Name: %s\n", a.AlbumName) if a.PrivateMeta != nil { fmt.Printf("PrivateMeta: %s\n", *a.PrivateMeta) } @@ -21,5 +24,6 @@ func PrintAlbum(a *model.Album) { if a.SharedMeta != nil { fmt.Printf("SharedMeta: %s\n", *a.SharedMeta) } - fmt.Printf("LastUpdatedAt: %d\n", a.LastUpdatedAt) + fmt.Printf("LastUpdatedAt: %d", a.LastUpdatedAt) + fmt.Printf("\n=======") } diff --git a/pkg/mappers.go b/pkg/mappers.go new file mode 100644 index 000000000..48846403f --- /dev/null +++ b/pkg/mappers.go @@ -0,0 +1,55 @@ +package pkg + +import ( + "cli-go/internal/api" + enteCrypto "cli-go/internal/crypto" + "cli-go/pkg/model" + "context" + "log" +) + +func (c *ClICtrl) mapCollectionToAlbum(ctx context.Context, collection api.Collection) (*model.Album, error) { + var album model.Album + userID := ctx.Value("user_id").(int64) + collectionKey, err := c.KeyHolder.GetCollectionKey(ctx, collection) + album.OwnerID = collection.Owner.ID + album.ID = collection.ID + album.IsShared = collection.Owner.ID != userID + album.AlbumKey = *model.MakeEncString(collectionKey, c.CliKey) + album.LastUpdatedAt = collection.UpdationTime + album.IsDeleted = collection.IsDeleted + if err != nil { + return nil, err + } + name, nameErr := enteCrypto.SecretBoxOpenBase64(collection.EncryptedName, collection.NameDecryptionNonce, collectionKey) + if nameErr != nil { + log.Fatalf("failed to decrypt collection name: %v", nameErr) + } + album.AlbumName = string(name) + + if collection.MagicMetadata != nil { + _, encodedJsonBytes, err := enteCrypto.DecryptChaChaBase64(collection.MagicMetadata.Data, collectionKey, collection.MagicMetadata.Header) + if err != nil { + return nil, err + } + var val = string(encodedJsonBytes) + album.PrivateMeta = &val + } + if collection.PublicMagicMetadata != nil { + _, encodedJsonBytes, err := enteCrypto.DecryptChaChaBase64(collection.PublicMagicMetadata.Data, collectionKey, collection.PublicMagicMetadata.Header) + if err != nil { + return nil, err + } + var val = string(encodedJsonBytes) + album.PublicMeta = &val + } + if album.IsShared && collection.SharedMagicMetadata != nil { + _, encodedJsonBytes, err := enteCrypto.DecryptChaChaBase64(collection.SharedMagicMetadata.Data, collectionKey, collection.SharedMagicMetadata.Header) + if err != nil { + return nil, err + } + var val = string(encodedJsonBytes) + album.SharedMeta = &val + } + return &album, nil +} diff --git a/pkg/mappers/album_mapper.go b/pkg/mappers/album_mapper.go deleted file mode 100644 index bff8365e0..000000000 --- a/pkg/mappers/album_mapper.go +++ /dev/null @@ -1,10 +0,0 @@ -package mappers - -import ( - "cli-go/internal/api" - "cli-go/pkg/model" -) - -func MapCollectionToAlbum(collection *api.Collection, collectionKey, cliKey []byte) (*model.Album, error) { - panic("not implemented") -} diff --git a/pkg/model/album.go b/pkg/model/album.go index 0f7272d2b..dc4b1742c 100644 --- a/pkg/model/album.go +++ b/pkg/model/album.go @@ -3,6 +3,8 @@ package model type Album struct { ID int64 `json:"id"` OwnerID int64 `json:"ownerID"` + IsShared bool `json:"isShared"` + IsDeleted bool `json:"isDeleted"` AlbumName string `json:"albumName"` AlbumKey EncString `json:"albumKey"` PublicMeta *string `json:"publicMeta"` From a79c8b34ff186a98c32d5326b38f9ac90d846a2a Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sun, 24 Sep 2023 23:44:08 +0530 Subject: [PATCH 057/183] Fix mapping for early collections --- pkg/mappers.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/pkg/mappers.go b/pkg/mappers.go index 48846403f..273c557f7 100644 --- a/pkg/mappers.go +++ b/pkg/mappers.go @@ -11,22 +11,28 @@ import ( func (c *ClICtrl) mapCollectionToAlbum(ctx context.Context, collection api.Collection) (*model.Album, error) { var album model.Album userID := ctx.Value("user_id").(int64) - collectionKey, err := c.KeyHolder.GetCollectionKey(ctx, collection) album.OwnerID = collection.Owner.ID album.ID = collection.ID album.IsShared = collection.Owner.ID != userID - album.AlbumKey = *model.MakeEncString(collectionKey, c.CliKey) album.LastUpdatedAt = collection.UpdationTime album.IsDeleted = collection.IsDeleted + collectionKey, err := c.KeyHolder.GetCollectionKey(ctx, collection) if err != nil { return nil, err } - name, nameErr := enteCrypto.SecretBoxOpenBase64(collection.EncryptedName, collection.NameDecryptionNonce, collectionKey) - if nameErr != nil { - log.Fatalf("failed to decrypt collection name: %v", nameErr) + album.AlbumKey = *model.MakeEncString(collectionKey, c.CliKey) + var name string + if collection.EncryptedName != "" { + decrName, err := enteCrypto.SecretBoxOpenBase64(collection.EncryptedName, collection.NameDecryptionNonce, collectionKey) + if err != nil { + log.Fatalf("failed to decrypt collection name: %v", err) + } + name = string(decrName) + } else { + // Early beta users (friends & family) might have collections without encrypted names + name = collection.Name } - album.AlbumName = string(name) - + album.AlbumName = name if collection.MagicMetadata != nil { _, encodedJsonBytes, err := enteCrypto.DecryptChaChaBase64(collection.MagicMetadata.Data, collectionKey, collection.MagicMetadata.Header) if err != nil { From ba7d0c4493c81ba78784f7d02db002bd0a88dd5a Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 25 Sep 2023 01:40:14 +0530 Subject: [PATCH 058/183] Fix API request --- internal/api/collection.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/internal/api/collection.go b/internal/api/collection.go index 2b24145a6..41e387be5 100644 --- a/internal/api/collection.go +++ b/internal/api/collection.go @@ -1,16 +1,19 @@ package api -import "context" +import ( + "context" + "strconv" +) -func (c *Client) GetCollections(ctx context.Context, sinceTime int) ([]Collection, error) { +func (c *Client) GetCollections(ctx context.Context, sinceTime int64) ([]Collection, error) { var res struct { Collections []Collection `json:"collections"` } r, err := c.restClient.R(). SetContext(ctx). - SetQueryParam("since", "0"). + SetQueryParam("sinceTime", strconv.FormatInt(sinceTime, 10)). SetResult(&res). - Get("/collections") + Get("/collections/v2") if r.IsError() { return nil, &ApiError{ StatusCode: r.StatusCode(), From 4eb3483e8781ce9fd07a34b48e6012ad930c1d44 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 25 Sep 2023 01:50:52 +0530 Subject: [PATCH 059/183] Update stores --- pkg/model/constants.go | 14 ++++++++++++++ pkg/remote_sync.go | 8 +++----- 2 files changed, 17 insertions(+), 5 deletions(-) create mode 100644 pkg/model/constants.go diff --git a/pkg/model/constants.go b/pkg/model/constants.go new file mode 100644 index 000000000..65db2b450 --- /dev/null +++ b/pkg/model/constants.go @@ -0,0 +1,14 @@ +package model + +type PhotosStore string + +const ( + KVConfig PhotosStore = "kvConfig" + RemoteAlbums PhotosStore = "remoteAlbums" + RemoteFiles PhotosStore = "remoteFiles" +) + +const ( + CollectionsSyncKey = "lastCollectionSync" + CollectionsFileSyncKeyFmt = "collectionFilesSync-%d" +) diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index 9187fb55d..b4747d05b 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -31,16 +31,14 @@ func (c *ClICtrl) buildRequestContext(ctx context.Context, account model.Account return ctx } -var dataCategories = []string{"remote-collections", "local-collections", "remote-files", "local-files", "remote-collection-removed", "remote-files-removed"} - func createDataBuckets(db *bolt.DB, account model.Account) error { return db.Update(func(tx *bolt.Tx) error { - dataBucket, err := tx.CreateBucketIfNotExists([]byte(account.DataBucket())) + dataBucket, err := tx.CreateBucketIfNotExists([]byte(account.AccountKey())) if err != nil { return fmt.Errorf("create bucket: %s", err) } - for _, category := range dataCategories { - _, err := dataBucket.CreateBucketIfNotExists([]byte(fmt.Sprintf(category))) + for _, subBucket := range []model.PhotosStore{model.KVConfig, model.RemoteAlbums, model.RemoteFiles} { + _, err := dataBucket.CreateBucketIfNotExists([]byte(subBucket)) if err != nil { return err } From a800ce745af5a5f859310f00caf4d55bff711d65 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 25 Sep 2023 01:51:11 +0530 Subject: [PATCH 060/183] Persist and use collection last sync time --- pkg/bolt_db.go | 16 -------------- pkg/collections.go | 25 +++++++++++++++++++-- pkg/store.go | 55 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 18 deletions(-) delete mode 100644 pkg/bolt_db.go create mode 100644 pkg/store.go diff --git a/pkg/bolt_db.go b/pkg/bolt_db.go deleted file mode 100644 index 533ac71b1..000000000 --- a/pkg/bolt_db.go +++ /dev/null @@ -1,16 +0,0 @@ -package pkg - -import ( - "log" - "time" - - bolt "go.etcd.io/bbolt" -) - -func GetDB(path string) (*bolt.DB, error) { - db, err := bolt.Open(path, 0600, &bolt.Options{Timeout: 1 * time.Second}) - if err != nil { - log.Fatal(err) - } - return db, err -} diff --git a/pkg/collections.go b/pkg/collections.go index 72e26eeb8..88dddf1d3 100644 --- a/pkg/collections.go +++ b/pkg/collections.go @@ -5,20 +5,41 @@ import ( "cli-go/pkg/model" "context" "fmt" + "strconv" ) func (c *ClICtrl) syncRemoteCollections(ctx context.Context, info model.Account) error { - collections, err := c.Client.GetCollections(ctx, 0) + valueBytes, err := c.GetConfigValue(ctx, model.CollectionsSyncKey) + if err != nil { + return fmt.Errorf("failed to get last sync time: %s", err) + } + var lastSyncTime int64 + if valueBytes != nil { + lastSyncTime, err = strconv.ParseInt(string(valueBytes), 10, 64) + if err != nil { + return err + } + } + collections, err := c.Client.GetCollections(ctx, lastSyncTime) if err != nil { return fmt.Errorf("failed to get collections: %s", err) } - + maxUpdated := lastSyncTime for _, collection := range collections { album, err2 := c.mapCollectionToAlbum(ctx, collection) if err2 != nil { return err2 } + if album.LastUpdatedAt > maxUpdated { + maxUpdated = album.LastUpdatedAt + } debuglog.PrintAlbum(album) } + if maxUpdated > lastSyncTime { + err = c.PutConfigValue(ctx, model.CollectionsSyncKey, []byte(strconv.FormatInt(maxUpdated, 10))) + if err != nil { + return fmt.Errorf("failed to update last sync time: %s", err) + } + } return nil } diff --git a/pkg/store.go b/pkg/store.go new file mode 100644 index 000000000..c75bbde51 --- /dev/null +++ b/pkg/store.go @@ -0,0 +1,55 @@ +package pkg + +import ( + "cli-go/pkg/model" + "context" + "fmt" + "log" + "time" + + bolt "go.etcd.io/bbolt" +) + +func GetDB(path string) (*bolt.DB, error) { + db, err := bolt.Open(path, 0600, &bolt.Options{Timeout: 1 * time.Second}) + if err != nil { + log.Fatal(err) + } + return db, err +} + +func (c *ClICtrl) GetConfigValue(ctx context.Context, key string) ([]byte, error) { + var value []byte + err := c.DB.View(func(tx *bolt.Tx) error { + kvBucket, err := getAccountStore(ctx, tx, model.KVConfig) + if err != nil { + return err + } + value = kvBucket.Get([]byte(key)) + return nil + }) + return value, err +} + +func (c *ClICtrl) PutConfigValue(ctx context.Context, key string, value []byte) error { + return c.DB.Update(func(tx *bolt.Tx) error { + kvBucket, err := getAccountStore(ctx, tx, model.KVConfig) + if err != nil { + return err + } + return kvBucket.Put([]byte(key), value) + }) +} + +func getAccountStore(ctx context.Context, tx *bolt.Tx, storeType model.PhotosStore) (*bolt.Bucket, error) { + accountId := ctx.Value("account_id").(string) + accountBucket := tx.Bucket([]byte(accountId)) + if accountBucket == nil { + return nil, fmt.Errorf("account bucket not found") + } + store := accountBucket.Bucket([]byte(storeType)) + if store == nil { + return nil, fmt.Errorf("store %s not found", storeType) + } + return store, nil +} From 47fbe0140bc43739ec4d452a8723a16cd2b6d2f1 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 25 Sep 2023 01:52:03 +0530 Subject: [PATCH 061/183] Ignore deleted collections on first sync --- pkg/collections.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/collections.go b/pkg/collections.go index 88dddf1d3..d78d8a2da 100644 --- a/pkg/collections.go +++ b/pkg/collections.go @@ -26,6 +26,9 @@ func (c *ClICtrl) syncRemoteCollections(ctx context.Context, info model.Account) } maxUpdated := lastSyncTime for _, collection := range collections { + if lastSyncTime == 0 && collection.IsDeleted { + continue + } album, err2 := c.mapCollectionToAlbum(ctx, collection) if err2 != nil { return err2 From ed3d4edaadd297d69e3a9a16c61e12f2e15c291d Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 25 Sep 2023 08:10:15 +0530 Subject: [PATCH 062/183] Store remote albums in local db --- pkg/collections.go | 6 ++++++ pkg/store.go | 9 +++++++++ utils/encoding/encoding.go | 9 +++++++++ 3 files changed, 24 insertions(+) diff --git a/pkg/collections.go b/pkg/collections.go index d78d8a2da..85d858259 100644 --- a/pkg/collections.go +++ b/pkg/collections.go @@ -3,6 +3,7 @@ package pkg import ( debuglog "cli-go/pkg/log" "cli-go/pkg/model" + "cli-go/utils/encoding" "context" "fmt" "strconv" @@ -36,6 +37,11 @@ func (c *ClICtrl) syncRemoteCollections(ctx context.Context, info model.Account) if album.LastUpdatedAt > maxUpdated { maxUpdated = album.LastUpdatedAt } + albumJson := encoding.MustMarshalJSON(album) + err := c.PutValue(ctx, model.RemoteAlbums, []byte(strconv.FormatInt(album.ID, 10)), albumJson) + if err != nil { + return err + } debuglog.PrintAlbum(album) } if maxUpdated > lastSyncTime { diff --git a/pkg/store.go b/pkg/store.go index c75bbde51..93456c3b3 100644 --- a/pkg/store.go +++ b/pkg/store.go @@ -40,6 +40,15 @@ func (c *ClICtrl) PutConfigValue(ctx context.Context, key string, value []byte) return kvBucket.Put([]byte(key), value) }) } +func (c *ClICtrl) PutValue(ctx context.Context, store model.PhotosStore, key []byte, value []byte) error { + return c.DB.Update(func(tx *bolt.Tx) error { + kvBucket, err := getAccountStore(ctx, tx, store) + if err != nil { + return err + } + return kvBucket.Put(key, value) + }) +} func getAccountStore(ctx context.Context, tx *bolt.Tx, storeType model.PhotosStore) (*bolt.Bucket, error) { accountId := ctx.Value("account_id").(string) diff --git a/utils/encoding/encoding.go b/utils/encoding/encoding.go index b5cec0b5f..a88eceff7 100644 --- a/utils/encoding/encoding.go +++ b/utils/encoding/encoding.go @@ -2,6 +2,7 @@ package encoding import ( "encoding/base64" + "encoding/json" ) func DecodeBase64(s string) []byte { @@ -15,3 +16,11 @@ func DecodeBase64(s string) []byte { func EncodeBase64(b []byte) string { return base64.StdEncoding.EncodeToString(b) } + +func MustMarshalJSON(v interface{}) []byte { + b, err := json.Marshal(v) + if err != nil { + panic(err) + } + return b +} From f91a424e44a033b5d2d7d10c4a7e0f5774214236 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 25 Sep 2023 08:17:29 +0530 Subject: [PATCH 063/183] Refactor --- pkg/collections.go | 27 ++++++++++----------------- pkg/remote_sync.go | 2 +- pkg/store.go | 16 ++++++++++++++++ 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/pkg/collections.go b/pkg/collections.go index 85d858259..246f9eb37 100644 --- a/pkg/collections.go +++ b/pkg/collections.go @@ -9,17 +9,10 @@ import ( "strconv" ) -func (c *ClICtrl) syncRemoteCollections(ctx context.Context, info model.Account) error { - valueBytes, err := c.GetConfigValue(ctx, model.CollectionsSyncKey) - if err != nil { - return fmt.Errorf("failed to get last sync time: %s", err) - } - var lastSyncTime int64 - if valueBytes != nil { - lastSyncTime, err = strconv.ParseInt(string(valueBytes), 10, 64) - if err != nil { - return err - } +func (c *ClICtrl) fetchRemoteCollections(ctx context.Context, info model.Account) error { + lastSyncTime, err2 := c.GetInt64ConfigValue(ctx, model.CollectionsSyncKey) + if err2 != nil { + return err2 } collections, err := c.Client.GetCollections(ctx, lastSyncTime) if err != nil { @@ -30,17 +23,17 @@ func (c *ClICtrl) syncRemoteCollections(ctx context.Context, info model.Account) if lastSyncTime == 0 && collection.IsDeleted { continue } - album, err2 := c.mapCollectionToAlbum(ctx, collection) - if err2 != nil { - return err2 + album, mapErr := c.mapCollectionToAlbum(ctx, collection) + if mapErr != nil { + return mapErr } if album.LastUpdatedAt > maxUpdated { maxUpdated = album.LastUpdatedAt } albumJson := encoding.MustMarshalJSON(album) - err := c.PutValue(ctx, model.RemoteAlbums, []byte(strconv.FormatInt(album.ID, 10)), albumJson) - if err != nil { - return err + putErr := c.PutValue(ctx, model.RemoteAlbums, []byte(strconv.FormatInt(album.ID, 10)), albumJson) + if putErr != nil { + return putErr } debuglog.PrintAlbum(album) } diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index b4747d05b..45e1598d2 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -21,7 +21,7 @@ func (c *ClICtrl) SyncAccount(account model.Account) error { return err } c.Client.AddToken(account.AccountKey(), base64.URLEncoding.EncodeToString(secretInfo.Token)) - return c.syncRemoteCollections(ctx, account) + return c.fetchRemoteCollections(ctx, account) } func (c *ClICtrl) buildRequestContext(ctx context.Context, account model.Account) context.Context { diff --git a/pkg/store.go b/pkg/store.go index 93456c3b3..56adf5a87 100644 --- a/pkg/store.go +++ b/pkg/store.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "log" + "strconv" "time" bolt "go.etcd.io/bbolt" @@ -31,6 +32,21 @@ func (c *ClICtrl) GetConfigValue(ctx context.Context, key string) ([]byte, error return value, err } +func (c *ClICtrl) GetInt64ConfigValue(ctx context.Context, key string) (int64, error) { + value, err := c.GetConfigValue(ctx, key) + if err != nil { + return 0, err + } + var result int64 + if value != nil { + result, err = strconv.ParseInt(string(value), 10, 64) + if err != nil { + return 0, err + } + } + return result, nil +} + func (c *ClICtrl) PutConfigValue(ctx context.Context, key string, value []byte) error { return c.DB.Update(func(tx *bolt.Tx) error { kvBucket, err := getAccountStore(ctx, tx, model.KVConfig) From cab929da2b39be3c1c652a236a9e0ff71cf19c5b Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 25 Sep 2023 09:58:15 +0530 Subject: [PATCH 064/183] Add log prefix --- pkg/remote_sync.go | 7 +++++-- pkg/sync.go | 7 ++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index 45e1598d2..c3e0831cb 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -10,7 +10,6 @@ import ( ) func (c *ClICtrl) SyncAccount(account model.Account) error { - log.SetPrefix(fmt.Sprintf("[%s] ", account.Email)) secretInfo, err := c.KeyHolder.LoadSecrets(account, c.CliKey) if err != nil { return err @@ -21,7 +20,11 @@ func (c *ClICtrl) SyncAccount(account model.Account) error { return err } c.Client.AddToken(account.AccountKey(), base64.URLEncoding.EncodeToString(secretInfo.Token)) - return c.fetchRemoteCollections(ctx, account) + err = c.fetchRemoteCollections(ctx, account) + if err != nil { + log.Printf("Error fetching collections: %s", err) + } + return nil } func (c *ClICtrl) buildRequestContext(ctx context.Context, account model.Account) context.Context { diff --git a/pkg/sync.go b/pkg/sync.go index 3476d2e8e..194515bbf 100644 --- a/pkg/sync.go +++ b/pkg/sync.go @@ -3,6 +3,7 @@ package pkg import ( "context" "fmt" + "log" ) func (c *ClICtrl) StartSync() error { @@ -15,12 +16,16 @@ func (c *ClICtrl) StartSync() error { return nil } for _, account := range accounts { - fmt.Printf("Syncing account %s\n", account.Email) + log.SetPrefix(fmt.Sprintf("[%s-%s] ", account.App, account.Email)) + log.Println("start sync") err = c.SyncAccount(account) if err != nil { fmt.Printf("Error syncing account %s: %s\n", account.Email, err) return err + } else { + log.Println("sync done") } + } return nil } From 8aafeccd1d3e793b620fe36addd7fdfd20f93dc7 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 25 Sep 2023 11:53:12 +0530 Subject: [PATCH 065/183] Add support for persisting remote files metadata --- internal/api/collection.go | 21 ++++++++++- internal/api/files.go | 1 + pkg/collections.go | 72 ++++++++++++++++++++++++++++++++++++++ pkg/model/constants.go | 6 ++-- pkg/remote_sync.go | 4 +++ pkg/store.go | 43 +++++++++++++++-------- 6 files changed, 129 insertions(+), 18 deletions(-) create mode 100644 internal/api/files.go diff --git a/internal/api/collection.go b/internal/api/collection.go index 41e387be5..6af702381 100644 --- a/internal/api/collection.go +++ b/internal/api/collection.go @@ -20,6 +20,25 @@ func (c *Client) GetCollections(ctx context.Context, sinceTime int64) ([]Collect Message: r.String(), } } - return res.Collections, err } + +func (c *Client) GetFiles(ctx context.Context, collectionID, sinceTime int64) ([]File, bool, error) { + var res struct { + Files []File `json:"diff"` + HasMore bool `json:"hasMore"` + } + r, err := c.restClient.R(). + SetContext(ctx). + SetQueryParam("sinceTime", strconv.FormatInt(sinceTime, 10)). + SetQueryParam("collectionID", strconv.FormatInt(collectionID, 10)). + SetResult(&res). + Get("/collections/v2/diff") + if r.IsError() { + return nil, false, &ApiError{ + StatusCode: r.StatusCode(), + Message: r.String(), + } + } + return res.Files, res.HasMore, err +} diff --git a/internal/api/files.go b/internal/api/files.go new file mode 100644 index 000000000..778f64ec1 --- /dev/null +++ b/internal/api/files.go @@ -0,0 +1 @@ +package api diff --git a/pkg/collections.go b/pkg/collections.go index 246f9eb37..943d7e084 100644 --- a/pkg/collections.go +++ b/pkg/collections.go @@ -5,7 +5,9 @@ import ( "cli-go/pkg/model" "cli-go/utils/encoding" "context" + "encoding/json" "fmt" + "log" "strconv" ) @@ -45,3 +47,73 @@ func (c *ClICtrl) fetchRemoteCollections(ctx context.Context, info model.Account } return nil } + +func (c *ClICtrl) fetchRemoteFiles(ctx context.Context, info model.Account) error { + albums, err := c.getRemoteAlbums(ctx) + if err != nil { + return err + } + for _, album := range albums { + if album.IsDeleted { + log.Printf("Skipping album %s as it is deleted", album.AlbumName) + continue + } + lastSyncTime, lastSyncTimeErr := c.GetInt64ConfigValue(ctx, fmt.Sprintf(model.CollectionsFileSyncKeyFmt, album.ID)) + if lastSyncTimeErr != nil { + return lastSyncTimeErr + } + isFirstSync := lastSyncTime == 0 + for { + if lastSyncTime == album.LastUpdatedAt { + break + } + files, hasMore, err := c.Client.GetFiles(ctx, album.ID, lastSyncTime) + if err != nil { + return err + } + maxUpdated := lastSyncTime + for _, file := range files { + if file.UpdationTime > maxUpdated { + maxUpdated = file.UpdationTime + } + if isFirstSync && file.IsDeleted { + // on first sync, no need to sync delete markers + continue + } + fileJson := encoding.MustMarshalJSON(file) + putErr := c.PutValue(ctx, model.RemoteFiles, []byte(strconv.FormatInt(file.ID, 10)), fileJson) + if putErr != nil { + return putErr + } + } + if !hasMore { + maxUpdated = album.LastUpdatedAt + } + if maxUpdated > lastSyncTime || !hasMore { + err = c.PutConfigValue(ctx, fmt.Sprintf(model.CollectionsFileSyncKeyFmt, album.ID), []byte(strconv.FormatInt(maxUpdated, 10))) + if err != nil { + return fmt.Errorf("failed to update last sync time: %s", err) + } else { + lastSyncTime = maxUpdated + } + } + } + } + return nil +} +func (c *ClICtrl) getRemoteAlbums(ctx context.Context) ([]model.Album, error) { + albums := make([]model.Album, 0) + albumBytes, err := c.GetAllValues(ctx, model.RemoteAlbums) + if err != nil { + return nil, err + } + for _, albumJson := range albumBytes { + album := model.Album{} + err = json.Unmarshal(albumJson, &album) + if err != nil { + return nil, err + } + albums = append(albums, album) + } + return albums, nil +} diff --git a/pkg/model/constants.go b/pkg/model/constants.go index 65db2b450..a891a964b 100644 --- a/pkg/model/constants.go +++ b/pkg/model/constants.go @@ -3,9 +3,9 @@ package model type PhotosStore string const ( - KVConfig PhotosStore = "kvConfig" - RemoteAlbums PhotosStore = "remoteAlbums" - RemoteFiles PhotosStore = "remoteFiles" + KVConfig PhotosStore = "akvConfig" + RemoteAlbums PhotosStore = "aremoteAlbums" + RemoteFiles PhotosStore = "aremoteFiles" ) const ( diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index c3e0831cb..c31a44c46 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -24,6 +24,10 @@ func (c *ClICtrl) SyncAccount(account model.Account) error { if err != nil { log.Printf("Error fetching collections: %s", err) } + err = c.fetchRemoteFiles(ctx, account) + if err != nil { + log.Printf("Error fetching files: %s", err) + } return nil } diff --git a/pkg/store.go b/pkg/store.go index 56adf5a87..6ed09697b 100644 --- a/pkg/store.go +++ b/pkg/store.go @@ -19,19 +19,6 @@ func GetDB(path string) (*bolt.DB, error) { return db, err } -func (c *ClICtrl) GetConfigValue(ctx context.Context, key string) ([]byte, error) { - var value []byte - err := c.DB.View(func(tx *bolt.Tx) error { - kvBucket, err := getAccountStore(ctx, tx, model.KVConfig) - if err != nil { - return err - } - value = kvBucket.Get([]byte(key)) - return nil - }) - return value, err -} - func (c *ClICtrl) GetInt64ConfigValue(ctx context.Context, key string) (int64, error) { value, err := c.GetConfigValue(ctx, key) if err != nil { @@ -47,6 +34,35 @@ func (c *ClICtrl) GetInt64ConfigValue(ctx context.Context, key string) (int64, e return result, nil } +func (c *ClICtrl) GetConfigValue(ctx context.Context, key string) ([]byte, error) { + var value []byte + err := c.DB.View(func(tx *bolt.Tx) error { + kvBucket, err := getAccountStore(ctx, tx, model.KVConfig) + if err != nil { + return err + } + value = kvBucket.Get([]byte(key)) + return nil + }) + return value, err +} + +func (c *ClICtrl) GetAllValues(ctx context.Context, store model.PhotosStore) ([][]byte, error) { + result := make([][]byte, 0) + err := c.DB.View(func(tx *bolt.Tx) error { + kvBucket, err := getAccountStore(ctx, tx, store) + if err != nil { + return err + } + kvBucket.ForEach(func(k, v []byte) error { + result = append(result, v) + return nil + }) + return nil + }) + return result, err +} + func (c *ClICtrl) PutConfigValue(ctx context.Context, key string, value []byte) error { return c.DB.Update(func(tx *bolt.Tx) error { kvBucket, err := getAccountStore(ctx, tx, model.KVConfig) @@ -65,7 +81,6 @@ func (c *ClICtrl) PutValue(ctx context.Context, store model.PhotosStore, key []b return kvBucket.Put(key, value) }) } - func getAccountStore(ctx context.Context, tx *bolt.Tx, storeType model.PhotosStore) (*bolt.Bucket, error) { accountId := ctx.Value("account_id").(string) accountBucket := tx.Bucket([]byte(accountId)) From 920bd0c7df47bc63b373818309e0a59198112304 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 25 Sep 2023 12:14:41 +0530 Subject: [PATCH 066/183] Remove unused param --- pkg/collections.go | 4 ++-- pkg/remote_sync.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/collections.go b/pkg/collections.go index 943d7e084..11395bc5a 100644 --- a/pkg/collections.go +++ b/pkg/collections.go @@ -11,7 +11,7 @@ import ( "strconv" ) -func (c *ClICtrl) fetchRemoteCollections(ctx context.Context, info model.Account) error { +func (c *ClICtrl) fetchRemoteCollections(ctx context.Context) error { lastSyncTime, err2 := c.GetInt64ConfigValue(ctx, model.CollectionsSyncKey) if err2 != nil { return err2 @@ -48,7 +48,7 @@ func (c *ClICtrl) fetchRemoteCollections(ctx context.Context, info model.Account return nil } -func (c *ClICtrl) fetchRemoteFiles(ctx context.Context, info model.Account) error { +func (c *ClICtrl) fetchRemoteFiles(ctx context.Context) error { albums, err := c.getRemoteAlbums(ctx) if err != nil { return err diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index c31a44c46..759b4661b 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -20,11 +20,11 @@ func (c *ClICtrl) SyncAccount(account model.Account) error { return err } c.Client.AddToken(account.AccountKey(), base64.URLEncoding.EncodeToString(secretInfo.Token)) - err = c.fetchRemoteCollections(ctx, account) + err = c.fetchRemoteCollections(ctx) if err != nil { log.Printf("Error fetching collections: %s", err) } - err = c.fetchRemoteFiles(ctx, account) + err = c.fetchRemoteFiles(ctx) if err != nil { log.Printf("Error fetching files: %s", err) } From f06d9ba5bf2a7f9ee791577b9b5c79821207dc1b Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 25 Sep 2023 12:15:04 +0530 Subject: [PATCH 067/183] undo test changes --- pkg/model/constants.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/model/constants.go b/pkg/model/constants.go index a891a964b..65db2b450 100644 --- a/pkg/model/constants.go +++ b/pkg/model/constants.go @@ -3,9 +3,9 @@ package model type PhotosStore string const ( - KVConfig PhotosStore = "akvConfig" - RemoteAlbums PhotosStore = "aremoteAlbums" - RemoteFiles PhotosStore = "aremoteFiles" + KVConfig PhotosStore = "kvConfig" + RemoteAlbums PhotosStore = "remoteAlbums" + RemoteFiles PhotosStore = "remoteFiles" ) const ( From 4294747d36a9b802eecc27eb24d56b5ce7e5de03 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 25 Sep 2023 13:30:07 +0530 Subject: [PATCH 068/183] Add logic to decrypt file metadata and persist --- pkg/collections.go | 6 ++++- pkg/mappers.go | 51 +++++++++++++++++++++++++++++++++++++++++ pkg/model/photo_file.go | 18 +++++++++++++++ 3 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 pkg/model/photo_file.go diff --git a/pkg/collections.go b/pkg/collections.go index 11395bc5a..4272ed910 100644 --- a/pkg/collections.go +++ b/pkg/collections.go @@ -80,7 +80,11 @@ func (c *ClICtrl) fetchRemoteFiles(ctx context.Context) error { // on first sync, no need to sync delete markers continue } - fileJson := encoding.MustMarshalJSON(file) + photoFile, err := c.mapApiFileToPhotoFile(ctx, album, file) + if err != nil { + return err + } + fileJson := encoding.MustMarshalJSON(photoFile) putErr := c.PutValue(ctx, model.RemoteFiles, []byte(strconv.FormatInt(file.ID, 10)), fileJson) if putErr != nil { return putErr diff --git a/pkg/mappers.go b/pkg/mappers.go index 273c557f7..cd9b223b4 100644 --- a/pkg/mappers.go +++ b/pkg/mappers.go @@ -4,7 +4,10 @@ import ( "cli-go/internal/api" enteCrypto "cli-go/internal/crypto" "cli-go/pkg/model" + "cli-go/utils/encoding" "context" + "encoding/json" + "errors" "log" ) @@ -59,3 +62,51 @@ func (c *ClICtrl) mapCollectionToAlbum(ctx context.Context, collection api.Colle } return &album, nil } + +func (c *ClICtrl) mapApiFileToPhotoFile(ctx context.Context, album model.Album, file api.File) (*model.PhotoFile, error) { + if file.IsDeleted { + return nil, errors.New("file is deleted") + } + albumKey := album.AlbumKey.MustDecrypt(c.CliKey) + fileKey, err := enteCrypto.SecretBoxOpen( + encoding.DecodeBase64(file.EncryptedKey), + encoding.DecodeBase64(file.KeyDecryptionNonce), + albumKey) + if err != nil { + return nil, err + } + var photoFile model.PhotoFile + photoFile.ID = file.ID + photoFile.Key = *model.MakeEncString(fileKey, c.CliKey) + photoFile.FileNonce = file.File.DecryptionHeader + photoFile.ThumbnailNonce = file.Thumbnail.DecryptionHeader + photoFile.OwnerID = file.OwnerID + if file.Info != nil { + photoFile.PhotoInfo = model.PhotoInfo{ + FileSize: file.Info.FileSize, + ThumbnailSize: file.Info.ThumbnailSize, + } + } + if file.Metadata.DecryptionHeader != "" { + _, encodedJsonBytes, err := enteCrypto.DecryptChaChaBase64(file.Metadata.EncryptedData, fileKey, file.Metadata.DecryptionHeader) + if err != nil { + return nil, err + } + err = json.Unmarshal(encodedJsonBytes, &photoFile.PrivateMetadata) + if err != nil { + return nil, err + } + } + if file.MagicMetadata != nil { + _, encodedJsonBytes, err := enteCrypto.DecryptChaChaBase64(file.MagicMetadata.Data, fileKey, file.MagicMetadata.Header) + if err != nil { + return nil, err + } + err = json.Unmarshal(encodedJsonBytes, &photoFile.PublicMetadata) + if err != nil { + return nil, err + } + } + return &photoFile, nil + +} diff --git a/pkg/model/photo_file.go b/pkg/model/photo_file.go new file mode 100644 index 000000000..467be6a2a --- /dev/null +++ b/pkg/model/photo_file.go @@ -0,0 +1,18 @@ +package model + +type PhotoFile struct { + ID int64 `json:"id"` + OwnerID int64 `json:"ownerID"` + Key EncString `json:"key"` + LastUpdateTime int64 `json:"lastUpdateTime"` + FileNonce string `json:"fileNonce"` + ThumbnailNonce string `json:"thumbnailNonce"` + PrivateMetadata map[string]interface{} `json:"privateMetadata"` + PublicMetadata map[string]interface{} `json:"publicMetadata"` + PhotoInfo PhotoInfo `` +} + +type PhotoInfo struct { + FileSize int64 `json:"fileSize,omitempty"` + ThumbnailSize int64 `json:"thumbSize,omitempty"` +} From 12ff1ee552cb4b906e747d263c3c5de796e4719e Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 25 Sep 2023 14:03:58 +0530 Subject: [PATCH 069/183] Store metadata as map for albums --- main.go | 2 +- pkg/collections.go | 6 ++++++ pkg/log/debug_print.go | 6 +++--- pkg/mappers.go | 20 +++++++++++++------- pkg/model/album.go | 20 ++++++++++---------- pkg/model/photo_file.go | 2 +- 6 files changed, 34 insertions(+), 22 deletions(-) diff --git a/main.go b/main.go index c5831e766..be12a1ef5 100644 --- a/main.go +++ b/main.go @@ -15,7 +15,7 @@ func main() { ctrl := pkg.ClICtrl{ Client: api.NewClient(api.Params{ Debug: false, - Host: "http://localhost:8080", + //Host: "http://localhost:8080", }), DB: db, CliKey: secrets.GetOrCreateClISecret(), diff --git a/pkg/collections.go b/pkg/collections.go index 4272ed910..d8cca3e4a 100644 --- a/pkg/collections.go +++ b/pkg/collections.go @@ -9,6 +9,7 @@ import ( "fmt" "log" "strconv" + "time" ) func (c *ClICtrl) fetchRemoteCollections(ctx context.Context) error { @@ -64,9 +65,14 @@ func (c *ClICtrl) fetchRemoteFiles(ctx context.Context) error { } isFirstSync := lastSyncTime == 0 for { + if lastSyncTime == album.LastUpdatedAt { break } + if !isFirstSync { + t := time.UnixMicro(lastSyncTime) + log.Printf("Fetching files for album %s from %v\n", album.AlbumName, t) + } files, hasMore, err := c.Client.GetFiles(ctx, album.ID, lastSyncTime) if err != nil { return err diff --git a/pkg/log/debug_print.go b/pkg/log/debug_print.go index 35ac29c38..d161e5c52 100644 --- a/pkg/log/debug_print.go +++ b/pkg/log/debug_print.go @@ -16,13 +16,13 @@ func PrintAlbum(a *model.Album) { } fmt.Printf(" Name: %s\n", a.AlbumName) if a.PrivateMeta != nil { - fmt.Printf("PrivateMeta: %s\n", *a.PrivateMeta) + fmt.Printf("PrivateMeta: %s\n", a.PrivateMeta) } if a.PublicMeta != nil { - fmt.Printf("PublicMeta: %s\n", *a.PublicMeta) + fmt.Printf("PublicMeta: %s\n", a.PublicMeta) } if a.SharedMeta != nil { - fmt.Printf("SharedMeta: %s\n", *a.SharedMeta) + fmt.Printf("SharedMeta: %s\n", a.SharedMeta) } fmt.Printf("LastUpdatedAt: %d", a.LastUpdatedAt) fmt.Printf("\n=======") diff --git a/pkg/mappers.go b/pkg/mappers.go index cd9b223b4..49c839964 100644 --- a/pkg/mappers.go +++ b/pkg/mappers.go @@ -41,24 +41,30 @@ func (c *ClICtrl) mapCollectionToAlbum(ctx context.Context, collection api.Colle if err != nil { return nil, err } - var val = string(encodedJsonBytes) - album.PrivateMeta = &val + err = json.Unmarshal(encodedJsonBytes, &album.PrivateMeta) + if err != nil { + return nil, err + } } if collection.PublicMagicMetadata != nil { _, encodedJsonBytes, err := enteCrypto.DecryptChaChaBase64(collection.PublicMagicMetadata.Data, collectionKey, collection.PublicMagicMetadata.Header) if err != nil { return nil, err } - var val = string(encodedJsonBytes) - album.PublicMeta = &val + err = json.Unmarshal(encodedJsonBytes, &album.PublicMeta) + if err != nil { + return nil, err + } } if album.IsShared && collection.SharedMagicMetadata != nil { _, encodedJsonBytes, err := enteCrypto.DecryptChaChaBase64(collection.SharedMagicMetadata.Data, collectionKey, collection.SharedMagicMetadata.Header) if err != nil { return nil, err } - var val = string(encodedJsonBytes) - album.SharedMeta = &val + err = json.Unmarshal(encodedJsonBytes, &album.SharedMeta) + if err != nil { + return nil, err + } } return &album, nil } @@ -82,7 +88,7 @@ func (c *ClICtrl) mapApiFileToPhotoFile(ctx context.Context, album model.Album, photoFile.ThumbnailNonce = file.Thumbnail.DecryptionHeader photoFile.OwnerID = file.OwnerID if file.Info != nil { - photoFile.PhotoInfo = model.PhotoInfo{ + photoFile.Info = model.PhotoInfo{ FileSize: file.Info.FileSize, ThumbnailSize: file.Info.ThumbnailSize, } diff --git a/pkg/model/album.go b/pkg/model/album.go index dc4b1742c..203fe1449 100644 --- a/pkg/model/album.go +++ b/pkg/model/album.go @@ -1,14 +1,14 @@ package model type Album struct { - ID int64 `json:"id"` - OwnerID int64 `json:"ownerID"` - IsShared bool `json:"isShared"` - IsDeleted bool `json:"isDeleted"` - AlbumName string `json:"albumName"` - AlbumKey EncString `json:"albumKey"` - PublicMeta *string `json:"publicMeta"` - PrivateMeta *string `json:"privateMeta"` - SharedMeta *string `json:"sharedMeta"` - LastUpdatedAt int64 `json:"lastUpdatedAt"` + ID int64 `json:"id"` + OwnerID int64 `json:"ownerID"` + IsShared bool `json:"isShared"` + IsDeleted bool `json:"isDeleted"` + AlbumName string `json:"albumName"` + AlbumKey EncString `json:"albumKey"` + PublicMeta map[string]interface{} `json:"publicMeta"` + PrivateMeta map[string]interface{} `json:"privateMeta"` + SharedMeta map[string]interface{} `json:"sharedMeta"` + LastUpdatedAt int64 `json:"lastUpdatedAt"` } diff --git a/pkg/model/photo_file.go b/pkg/model/photo_file.go index 467be6a2a..f84c1556a 100644 --- a/pkg/model/photo_file.go +++ b/pkg/model/photo_file.go @@ -9,7 +9,7 @@ type PhotoFile struct { ThumbnailNonce string `json:"thumbnailNonce"` PrivateMetadata map[string]interface{} `json:"privateMetadata"` PublicMetadata map[string]interface{} `json:"publicMetadata"` - PhotoInfo PhotoInfo `` + Info PhotoInfo `json:"info"` } type PhotoInfo struct { From 8b1064f4017cc142d92235c7f887fe88c7e2b3b3 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 25 Sep 2023 14:19:03 +0530 Subject: [PATCH 070/183] Update README.md --- README.md | 4 +++- pkg/collections.go | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b0fe07ad5..287a14b85 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -# cli tool for exporting ente photos +# cli tool for exporting data from ente.io + +#### You can configure multiple accounts for export ### Getting Started diff --git a/pkg/collections.go b/pkg/collections.go index d8cca3e4a..cf12c5e5d 100644 --- a/pkg/collections.go +++ b/pkg/collections.go @@ -1,7 +1,6 @@ package pkg import ( - debuglog "cli-go/pkg/log" "cli-go/pkg/model" "cli-go/utils/encoding" "context" @@ -38,7 +37,6 @@ func (c *ClICtrl) fetchRemoteCollections(ctx context.Context) error { if putErr != nil { return putErr } - debuglog.PrintAlbum(album) } if maxUpdated > lastSyncTime { err = c.PutConfigValue(ctx, model.CollectionsSyncKey, []byte(strconv.FormatInt(maxUpdated, 10))) From fc0abaff5a94ecd926450a0a1f05d4e263c6f9ba Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 25 Sep 2023 18:21:45 +0530 Subject: [PATCH 071/183] Fix mapping of metadata --- pkg/mappers.go | 12 +++++++++++- pkg/model/photo_file.go | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/mappers.go b/pkg/mappers.go index 49c839964..531fa9685 100644 --- a/pkg/mappers.go +++ b/pkg/mappers.go @@ -98,7 +98,7 @@ func (c *ClICtrl) mapApiFileToPhotoFile(ctx context.Context, album model.Album, if err != nil { return nil, err } - err = json.Unmarshal(encodedJsonBytes, &photoFile.PrivateMetadata) + err = json.Unmarshal(encodedJsonBytes, &photoFile.Metadata) if err != nil { return nil, err } @@ -108,6 +108,16 @@ func (c *ClICtrl) mapApiFileToPhotoFile(ctx context.Context, album model.Album, if err != nil { return nil, err } + err = json.Unmarshal(encodedJsonBytes, &photoFile.PrivateMetadata) + if err != nil { + return nil, err + } + } + if file.PubicMagicMetadata != nil { + _, encodedJsonBytes, err := enteCrypto.DecryptChaChaBase64(file.PubicMagicMetadata.Data, fileKey, file.PubicMagicMetadata.Header) + if err != nil { + return nil, err + } err = json.Unmarshal(encodedJsonBytes, &photoFile.PublicMetadata) if err != nil { return nil, err diff --git a/pkg/model/photo_file.go b/pkg/model/photo_file.go index f84c1556a..04234b7cf 100644 --- a/pkg/model/photo_file.go +++ b/pkg/model/photo_file.go @@ -7,6 +7,7 @@ type PhotoFile struct { LastUpdateTime int64 `json:"lastUpdateTime"` FileNonce string `json:"fileNonce"` ThumbnailNonce string `json:"thumbnailNonce"` + Metadata map[string]interface{} `json:"metadata"` PrivateMetadata map[string]interface{} `json:"privateMetadata"` PublicMetadata map[string]interface{} `json:"publicMetadata"` Info PhotoInfo `json:"info"` From bc7a8418ee8519336ecf3a2d13ae542ef7de7ee0 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 25 Sep 2023 19:10:36 +0530 Subject: [PATCH 072/183] Add support for downloading file --- internal/api/client.go | 22 +++++++++++++------- internal/api/files.go | 24 +++++++++++++++++++++ pkg/collections.go | 1 + pkg/download.go | 47 ++++++++++++++++++++++++++++++++++++++++++ pkg/remote_sync.go | 5 +++++ 5 files changed, 92 insertions(+), 7 deletions(-) create mode 100644 pkg/download.go diff --git a/internal/api/client.go b/internal/api/client.go index 7cd28c430..38756f7f9 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -19,6 +19,8 @@ var tokenMap map[string]string = make(map[string]string) type Client struct { restClient *resty.Client + // use separate client for downloading files + downloadClient *resty.Client } type Params struct { @@ -34,6 +36,7 @@ func readValueFromContext(ctx context.Context, key string) interface{} { func NewClient(p Params) *Client { enteAPI := resty.New() + if p.Trace { enteAPI.EnableTrace() } @@ -43,12 +46,7 @@ func NewClient(p Params) *Client { panic("app not set in context") } req.Header.Set(ClientPkgHeader, StringToApp(app.(string)).ClientPkg()) - accountId := readValueFromContext(req.Context(), "account_id") - if accountId != nil && accountId != "" { - if token, ok := tokenMap[accountId.(string)]; ok { - req.SetHeader(TokenHeader, token) - } - } + attachToken(req) return nil }) if p.Debug { @@ -68,7 +66,17 @@ func NewClient(p Params) *Client { enteAPI.SetBaseURL(EnteAPIEndpoint) } return &Client{ - restClient: enteAPI, + restClient: enteAPI, + downloadClient: resty.New(), + } +} + +func attachToken(req *resty.Request) { + accountId := readValueFromContext(req.Context(), "account_id") + if accountId != nil && accountId != "" { + if token, ok := tokenMap[accountId.(string)]; ok { + req.SetHeader(TokenHeader, token) + } } } diff --git a/internal/api/files.go b/internal/api/files.go index 778f64ec1..2e4af7701 100644 --- a/internal/api/files.go +++ b/internal/api/files.go @@ -1 +1,25 @@ package api + +import ( + "context" + "strconv" +) + +var ( + downloadHost = "https://files.ente.io/?fileID=" +) + +func (c *Client) DownloadFile(ctx context.Context, fileID int64, absolutePath string) error { + req := c.downloadClient.R(). + SetContext(ctx). + SetOutput(absolutePath) + attachToken(req) + r, err := req.Get(downloadHost + strconv.FormatInt(fileID, 10)) + if r.IsError() { + return &ApiError{ + StatusCode: r.StatusCode(), + Message: r.String(), + } + } + return err +} diff --git a/pkg/collections.go b/pkg/collections.go index cf12c5e5d..379114964 100644 --- a/pkg/collections.go +++ b/pkg/collections.go @@ -109,6 +109,7 @@ func (c *ClICtrl) fetchRemoteFiles(ctx context.Context) error { } return nil } + func (c *ClICtrl) getRemoteAlbums(ctx context.Context) ([]model.Album, error) { albums := make([]model.Album, 0) albumBytes, err := c.GetAllValues(ctx, model.RemoteAlbums) diff --git a/pkg/download.go b/pkg/download.go new file mode 100644 index 000000000..c747eae80 --- /dev/null +++ b/pkg/download.go @@ -0,0 +1,47 @@ +package pkg + +import ( + "cli-go/pkg/model" + "context" + "encoding/json" + "fmt" + "log" + "os" +) + +func (c *ClICtrl) initiateDownload(ctx context.Context) error { + files, err := c.getRemoteFiles(ctx) + if err != nil { + return err + } + dir, err := os.MkdirTemp("", "photos-download") + if err != nil { + return err + } + for _, file := range files { + downloadPath := fmt.Sprintf("%s/%d", dir, file.ID) + log.Printf("Downloading file %d to %s", file.ID, downloadPath) + //err = c.Client.DownloadFile(ctx, file.ID, downloadPath) + //if err != nil { + // return err + //} + } + return nil +} + +func (c *ClICtrl) getRemoteFiles(ctx context.Context) ([]model.PhotoFile, error) { + files := make([]model.PhotoFile, 0) + fileBytes, err := c.GetAllValues(ctx, model.RemoteFiles) + if err != nil { + return nil, err + } + for _, fileJson := range fileBytes { + file := model.PhotoFile{} + err = json.Unmarshal(fileJson, &file) + if err != nil { + return nil, err + } + files = append(files, file) + } + return files, nil +} diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index 759b4661b..f2e3c5c8c 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -28,6 +28,11 @@ func (c *ClICtrl) SyncAccount(account model.Account) error { if err != nil { log.Printf("Error fetching files: %s", err) } + downloadErr := c.initiateDownload(ctx) + if downloadErr != nil { + log.Printf("Error downloading files: %s", downloadErr) + return downloadErr + } return nil } From 2b2c3f47d821eeab40cb85ddfbc54d4eadf03d33 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 25 Sep 2023 19:12:56 +0530 Subject: [PATCH 073/183] Rename --- pkg/collections.go | 6 +++--- pkg/download.go | 6 +++--- pkg/log/debug_print.go | 2 +- pkg/mappers.go | 10 +++++----- pkg/model/album.go | 14 -------------- pkg/model/photo_file.go | 19 ------------------- pkg/model/remote.go | 32 ++++++++++++++++++++++++++++++++ 7 files changed, 44 insertions(+), 45 deletions(-) delete mode 100644 pkg/model/album.go delete mode 100644 pkg/model/photo_file.go create mode 100644 pkg/model/remote.go diff --git a/pkg/collections.go b/pkg/collections.go index 379114964..b2828d58d 100644 --- a/pkg/collections.go +++ b/pkg/collections.go @@ -110,14 +110,14 @@ func (c *ClICtrl) fetchRemoteFiles(ctx context.Context) error { return nil } -func (c *ClICtrl) getRemoteAlbums(ctx context.Context) ([]model.Album, error) { - albums := make([]model.Album, 0) +func (c *ClICtrl) getRemoteAlbums(ctx context.Context) ([]model.RemoteAlbum, error) { + albums := make([]model.RemoteAlbum, 0) albumBytes, err := c.GetAllValues(ctx, model.RemoteAlbums) if err != nil { return nil, err } for _, albumJson := range albumBytes { - album := model.Album{} + album := model.RemoteAlbum{} err = json.Unmarshal(albumJson, &album) if err != nil { return nil, err diff --git a/pkg/download.go b/pkg/download.go index c747eae80..4e82ecdeb 100644 --- a/pkg/download.go +++ b/pkg/download.go @@ -29,14 +29,14 @@ func (c *ClICtrl) initiateDownload(ctx context.Context) error { return nil } -func (c *ClICtrl) getRemoteFiles(ctx context.Context) ([]model.PhotoFile, error) { - files := make([]model.PhotoFile, 0) +func (c *ClICtrl) getRemoteFiles(ctx context.Context) ([]model.RemoteFile, error) { + files := make([]model.RemoteFile, 0) fileBytes, err := c.GetAllValues(ctx, model.RemoteFiles) if err != nil { return nil, err } for _, fileJson := range fileBytes { - file := model.PhotoFile{} + file := model.RemoteFile{} err = json.Unmarshal(fileJson, &file) if err != nil { return nil, err diff --git a/pkg/log/debug_print.go b/pkg/log/debug_print.go index d161e5c52..3cdc12097 100644 --- a/pkg/log/debug_print.go +++ b/pkg/log/debug_print.go @@ -7,7 +7,7 @@ import ( // This file contains functions that are used to print debug information to the console. -func PrintAlbum(a *model.Album) { +func PrintAlbum(a *model.RemoteAlbum) { fmt.Printf("=======\n") fmt.Printf("ID: %d\n", a.ID) fmt.Printf("OwnerID: %d\n", a.OwnerID) diff --git a/pkg/mappers.go b/pkg/mappers.go index 531fa9685..2aa67ee9e 100644 --- a/pkg/mappers.go +++ b/pkg/mappers.go @@ -11,8 +11,8 @@ import ( "log" ) -func (c *ClICtrl) mapCollectionToAlbum(ctx context.Context, collection api.Collection) (*model.Album, error) { - var album model.Album +func (c *ClICtrl) mapCollectionToAlbum(ctx context.Context, collection api.Collection) (*model.RemoteAlbum, error) { + var album model.RemoteAlbum userID := ctx.Value("user_id").(int64) album.OwnerID = collection.Owner.ID album.ID = collection.ID @@ -69,7 +69,7 @@ func (c *ClICtrl) mapCollectionToAlbum(ctx context.Context, collection api.Colle return &album, nil } -func (c *ClICtrl) mapApiFileToPhotoFile(ctx context.Context, album model.Album, file api.File) (*model.PhotoFile, error) { +func (c *ClICtrl) mapApiFileToPhotoFile(ctx context.Context, album model.RemoteAlbum, file api.File) (*model.RemoteFile, error) { if file.IsDeleted { return nil, errors.New("file is deleted") } @@ -81,14 +81,14 @@ func (c *ClICtrl) mapApiFileToPhotoFile(ctx context.Context, album model.Album, if err != nil { return nil, err } - var photoFile model.PhotoFile + var photoFile model.RemoteFile photoFile.ID = file.ID photoFile.Key = *model.MakeEncString(fileKey, c.CliKey) photoFile.FileNonce = file.File.DecryptionHeader photoFile.ThumbnailNonce = file.Thumbnail.DecryptionHeader photoFile.OwnerID = file.OwnerID if file.Info != nil { - photoFile.Info = model.PhotoInfo{ + photoFile.Info = model.Info{ FileSize: file.Info.FileSize, ThumbnailSize: file.Info.ThumbnailSize, } diff --git a/pkg/model/album.go b/pkg/model/album.go deleted file mode 100644 index 203fe1449..000000000 --- a/pkg/model/album.go +++ /dev/null @@ -1,14 +0,0 @@ -package model - -type Album struct { - ID int64 `json:"id"` - OwnerID int64 `json:"ownerID"` - IsShared bool `json:"isShared"` - IsDeleted bool `json:"isDeleted"` - AlbumName string `json:"albumName"` - AlbumKey EncString `json:"albumKey"` - PublicMeta map[string]interface{} `json:"publicMeta"` - PrivateMeta map[string]interface{} `json:"privateMeta"` - SharedMeta map[string]interface{} `json:"sharedMeta"` - LastUpdatedAt int64 `json:"lastUpdatedAt"` -} diff --git a/pkg/model/photo_file.go b/pkg/model/photo_file.go deleted file mode 100644 index 04234b7cf..000000000 --- a/pkg/model/photo_file.go +++ /dev/null @@ -1,19 +0,0 @@ -package model - -type PhotoFile struct { - ID int64 `json:"id"` - OwnerID int64 `json:"ownerID"` - Key EncString `json:"key"` - LastUpdateTime int64 `json:"lastUpdateTime"` - FileNonce string `json:"fileNonce"` - ThumbnailNonce string `json:"thumbnailNonce"` - Metadata map[string]interface{} `json:"metadata"` - PrivateMetadata map[string]interface{} `json:"privateMetadata"` - PublicMetadata map[string]interface{} `json:"publicMetadata"` - Info PhotoInfo `json:"info"` -} - -type PhotoInfo struct { - FileSize int64 `json:"fileSize,omitempty"` - ThumbnailSize int64 `json:"thumbSize,omitempty"` -} diff --git a/pkg/model/remote.go b/pkg/model/remote.go new file mode 100644 index 000000000..812fe5475 --- /dev/null +++ b/pkg/model/remote.go @@ -0,0 +1,32 @@ +package model + +type RemoteFile struct { + ID int64 `json:"id"` + OwnerID int64 `json:"ownerID"` + Key EncString `json:"key"` + LastUpdateTime int64 `json:"lastUpdateTime"` + FileNonce string `json:"fileNonce"` + ThumbnailNonce string `json:"thumbnailNonce"` + Metadata map[string]interface{} `json:"metadata"` + PrivateMetadata map[string]interface{} `json:"privateMetadata"` + PublicMetadata map[string]interface{} `json:"publicMetadata"` + Info Info `json:"info"` +} + +type Info struct { + FileSize int64 `json:"fileSize,omitempty"` + ThumbnailSize int64 `json:"thumbSize,omitempty"` +} + +type RemoteAlbum struct { + ID int64 `json:"id"` + OwnerID int64 `json:"ownerID"` + IsShared bool `json:"isShared"` + IsDeleted bool `json:"isDeleted"` + AlbumName string `json:"albumName"` + AlbumKey EncString `json:"albumKey"` + PublicMeta map[string]interface{} `json:"publicMeta"` + PrivateMeta map[string]interface{} `json:"privateMeta"` + SharedMeta map[string]interface{} `json:"sharedMeta"` + LastUpdatedAt int64 `json:"lastUpdatedAt"` +} From 985a7ccc7858fe4ebe60a7955d28e8ccaaa0c829 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Tue, 26 Sep 2023 17:04:34 +0530 Subject: [PATCH 074/183] Fix mapping of metadata updation Time --- pkg/{sync.go => export.go} | 0 pkg/mappers.go | 1 + 2 files changed, 1 insertion(+) rename pkg/{sync.go => export.go} (100%) diff --git a/pkg/sync.go b/pkg/export.go similarity index 100% rename from pkg/sync.go rename to pkg/export.go diff --git a/pkg/mappers.go b/pkg/mappers.go index 2aa67ee9e..dd09b8963 100644 --- a/pkg/mappers.go +++ b/pkg/mappers.go @@ -83,6 +83,7 @@ func (c *ClICtrl) mapApiFileToPhotoFile(ctx context.Context, album model.RemoteA } var photoFile model.RemoteFile photoFile.ID = file.ID + photoFile.LastUpdateTime = file.UpdationTime photoFile.Key = *model.MakeEncString(fileKey, c.CliKey) photoFile.FileNonce = file.File.DecryptionHeader photoFile.ThumbnailNonce = file.Thumbnail.DecryptionHeader From 9b50fbfa35e558e256e11474a6d10afd96bd1b66 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 27 Sep 2023 08:07:42 +0530 Subject: [PATCH 075/183] Minor rename --- pkg/log/debug_print.go | 29 ----------------------------- pkg/mappers.go | 18 +++++++++--------- pkg/secrets/key_holder.go | 6 +++--- pkg/sign_in.go | 14 +++++++------- 4 files changed, 19 insertions(+), 48 deletions(-) delete mode 100644 pkg/log/debug_print.go diff --git a/pkg/log/debug_print.go b/pkg/log/debug_print.go deleted file mode 100644 index 3cdc12097..000000000 --- a/pkg/log/debug_print.go +++ /dev/null @@ -1,29 +0,0 @@ -package debuglog - -import ( - "cli-go/pkg/model" - "fmt" -) - -// This file contains functions that are used to print debug information to the console. - -func PrintAlbum(a *model.RemoteAlbum) { - fmt.Printf("=======\n") - fmt.Printf("ID: %d\n", a.ID) - fmt.Printf("OwnerID: %d\n", a.OwnerID) - if a.IsShared { - fmt.Printf("Shared album") - } - fmt.Printf(" Name: %s\n", a.AlbumName) - if a.PrivateMeta != nil { - fmt.Printf("PrivateMeta: %s\n", a.PrivateMeta) - } - if a.PublicMeta != nil { - fmt.Printf("PublicMeta: %s\n", a.PublicMeta) - } - if a.SharedMeta != nil { - fmt.Printf("SharedMeta: %s\n", a.SharedMeta) - } - fmt.Printf("LastUpdatedAt: %d", a.LastUpdatedAt) - fmt.Printf("\n=======") -} diff --git a/pkg/mappers.go b/pkg/mappers.go index dd09b8963..510cad36d 100644 --- a/pkg/mappers.go +++ b/pkg/mappers.go @@ -2,7 +2,7 @@ package pkg import ( "cli-go/internal/api" - enteCrypto "cli-go/internal/crypto" + eCrypto "cli-go/internal/crypto" "cli-go/pkg/model" "cli-go/utils/encoding" "context" @@ -26,7 +26,7 @@ func (c *ClICtrl) mapCollectionToAlbum(ctx context.Context, collection api.Colle album.AlbumKey = *model.MakeEncString(collectionKey, c.CliKey) var name string if collection.EncryptedName != "" { - decrName, err := enteCrypto.SecretBoxOpenBase64(collection.EncryptedName, collection.NameDecryptionNonce, collectionKey) + decrName, err := eCrypto.SecretBoxOpenBase64(collection.EncryptedName, collection.NameDecryptionNonce, collectionKey) if err != nil { log.Fatalf("failed to decrypt collection name: %v", err) } @@ -37,7 +37,7 @@ func (c *ClICtrl) mapCollectionToAlbum(ctx context.Context, collection api.Colle } album.AlbumName = name if collection.MagicMetadata != nil { - _, encodedJsonBytes, err := enteCrypto.DecryptChaChaBase64(collection.MagicMetadata.Data, collectionKey, collection.MagicMetadata.Header) + _, encodedJsonBytes, err := eCrypto.DecryptChaChaBase64(collection.MagicMetadata.Data, collectionKey, collection.MagicMetadata.Header) if err != nil { return nil, err } @@ -47,7 +47,7 @@ func (c *ClICtrl) mapCollectionToAlbum(ctx context.Context, collection api.Colle } } if collection.PublicMagicMetadata != nil { - _, encodedJsonBytes, err := enteCrypto.DecryptChaChaBase64(collection.PublicMagicMetadata.Data, collectionKey, collection.PublicMagicMetadata.Header) + _, encodedJsonBytes, err := eCrypto.DecryptChaChaBase64(collection.PublicMagicMetadata.Data, collectionKey, collection.PublicMagicMetadata.Header) if err != nil { return nil, err } @@ -57,7 +57,7 @@ func (c *ClICtrl) mapCollectionToAlbum(ctx context.Context, collection api.Colle } } if album.IsShared && collection.SharedMagicMetadata != nil { - _, encodedJsonBytes, err := enteCrypto.DecryptChaChaBase64(collection.SharedMagicMetadata.Data, collectionKey, collection.SharedMagicMetadata.Header) + _, encodedJsonBytes, err := eCrypto.DecryptChaChaBase64(collection.SharedMagicMetadata.Data, collectionKey, collection.SharedMagicMetadata.Header) if err != nil { return nil, err } @@ -74,7 +74,7 @@ func (c *ClICtrl) mapApiFileToPhotoFile(ctx context.Context, album model.RemoteA return nil, errors.New("file is deleted") } albumKey := album.AlbumKey.MustDecrypt(c.CliKey) - fileKey, err := enteCrypto.SecretBoxOpen( + fileKey, err := eCrypto.SecretBoxOpen( encoding.DecodeBase64(file.EncryptedKey), encoding.DecodeBase64(file.KeyDecryptionNonce), albumKey) @@ -95,7 +95,7 @@ func (c *ClICtrl) mapApiFileToPhotoFile(ctx context.Context, album model.RemoteA } } if file.Metadata.DecryptionHeader != "" { - _, encodedJsonBytes, err := enteCrypto.DecryptChaChaBase64(file.Metadata.EncryptedData, fileKey, file.Metadata.DecryptionHeader) + _, encodedJsonBytes, err := eCrypto.DecryptChaChaBase64(file.Metadata.EncryptedData, fileKey, file.Metadata.DecryptionHeader) if err != nil { return nil, err } @@ -105,7 +105,7 @@ func (c *ClICtrl) mapApiFileToPhotoFile(ctx context.Context, album model.RemoteA } } if file.MagicMetadata != nil { - _, encodedJsonBytes, err := enteCrypto.DecryptChaChaBase64(file.MagicMetadata.Data, fileKey, file.MagicMetadata.Header) + _, encodedJsonBytes, err := eCrypto.DecryptChaChaBase64(file.MagicMetadata.Data, fileKey, file.MagicMetadata.Header) if err != nil { return nil, err } @@ -115,7 +115,7 @@ func (c *ClICtrl) mapApiFileToPhotoFile(ctx context.Context, album model.RemoteA } } if file.PubicMagicMetadata != nil { - _, encodedJsonBytes, err := enteCrypto.DecryptChaChaBase64(file.PubicMagicMetadata.Data, fileKey, file.PubicMagicMetadata.Header) + _, encodedJsonBytes, err := eCrypto.DecryptChaChaBase64(file.PubicMagicMetadata.Data, fileKey, file.PubicMagicMetadata.Header) if err != nil { return nil, err } diff --git a/pkg/secrets/key_holder.go b/pkg/secrets/key_holder.go index 3ceb5c779..336e3178f 100644 --- a/pkg/secrets/key_holder.go +++ b/pkg/secrets/key_holder.go @@ -2,7 +2,7 @@ package secrets import ( "cli-go/internal/api" - enteCrypto "cli-go/internal/crypto" + eCrypto "cli-go/internal/crypto" "cli-go/pkg/model" "cli-go/utils/encoding" "context" @@ -52,7 +52,7 @@ func (k *KeyHolder) GetCollectionKey(ctx context.Context, collection api.Collect accSecretInfo := k.GetAccountSecretInfo(ctx) userID := ctx.Value("user_id").(int64) if collection.Owner.ID == userID { - collKey, err := enteCrypto.SecretBoxOpen( + collKey, err := eCrypto.SecretBoxOpen( encoding.DecodeBase64(collection.EncryptedKey), encoding.DecodeBase64(collection.KeyDecryptionNonce), accSecretInfo.MasterKey) @@ -61,7 +61,7 @@ func (k *KeyHolder) GetCollectionKey(ctx context.Context, collection api.Collect } return collKey, nil } else { - collKey, err := enteCrypto.SealedBoxOpen(encoding.DecodeBase64(collection.EncryptedKey), + collKey, err := eCrypto.SealedBoxOpen(encoding.DecodeBase64(collection.EncryptedKey), accSecretInfo.PublicKey, accSecretInfo.SecretKey) if err != nil { return nil, fmt.Errorf("shared collection %d key drive failed %s", collection.ID, err) diff --git a/pkg/sign_in.go b/pkg/sign_in.go index f1e0b970f..ed26403ef 100644 --- a/pkg/sign_in.go +++ b/pkg/sign_in.go @@ -3,7 +3,7 @@ package pkg import ( "cli-go/internal" "cli-go/internal/api" - enteCrypto "cli-go/internal/crypto" + eCrypto "cli-go/internal/crypto" "cli-go/pkg/model" "cli-go/utils/encoding" "context" @@ -21,12 +21,12 @@ func (c *ClICtrl) signInViaPassword(ctx context.Context, email string, srpAttr * return nil, nil, flowErr } fmt.Println("\nPlease wait authenticating...") - keyEncKey, err := enteCrypto.DeriveArgonKey(password, srpAttr.KekSalt, srpAttr.MemLimit, srpAttr.OpsLimit) + keyEncKey, err := eCrypto.DeriveArgonKey(password, srpAttr.KekSalt, srpAttr.MemLimit, srpAttr.OpsLimit) if err != nil { fmt.Printf("error deriving key encryption key: %v", err) return nil, nil, err } - loginKey := enteCrypto.DeriveLoginKey(keyEncKey) + loginKey := eCrypto.DeriveLoginKey(keyEncKey) srpParams := srp.GetParams(4096) identify := []byte(srpAttr.SRPUserID.String()) @@ -70,7 +70,7 @@ func (c *ClICtrl) decryptAccSecretInfo( return nil, flowErr } fmt.Println("\nPlease wait authenticating...") - currentKeyEncKey, err = enteCrypto.DeriveArgonKey(password, + currentKeyEncKey, err = eCrypto.DeriveArgonKey(password, authResp.KeyAttributes.KEKSalt, authResp.KeyAttributes.MemLimit, authResp.KeyAttributes.OpsLimit) if err != nil { fmt.Printf("error deriving key encryption key: %v", err) @@ -82,7 +82,7 @@ func (c *ClICtrl) decryptAccSecretInfo( encryptedKey := encoding.DecodeBase64(authResp.KeyAttributes.EncryptedKey) encryptedKeyNonce := encoding.DecodeBase64(authResp.KeyAttributes.KeyDecryptionNonce) - masterKey, err = enteCrypto.SecretBoxOpen(encryptedKey, encryptedKeyNonce, currentKeyEncKey) + masterKey, err = eCrypto.SecretBoxOpen(encryptedKey, encryptedKeyNonce, currentKeyEncKey) if err != nil { if keyEncKey != nil { fmt.Printf("Failed to get key from keyEncryptionKey %s", err) @@ -92,7 +92,7 @@ func (c *ClICtrl) decryptAccSecretInfo( continue } } - secretKey, err = enteCrypto.SecretBoxOpen( + secretKey, err = eCrypto.SecretBoxOpen( encoding.DecodeBase64(authResp.KeyAttributes.EncryptedSecretKey), encoding.DecodeBase64(authResp.KeyAttributes.SecretKeyDecryptionNonce), masterKey, @@ -101,7 +101,7 @@ func (c *ClICtrl) decryptAccSecretInfo( fmt.Printf("error decrypting master key: %v", err) return nil, err } - tokenKey, err = enteCrypto.SealedBoxOpen( + tokenKey, err = eCrypto.SealedBoxOpen( encoding.DecodeBase64(authResp.EncryptedToken), publicKey, secretKey, From 86385a1a2fbc9dbb3fcff366e67f41ffd240fcf0 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 27 Sep 2023 11:02:36 +0530 Subject: [PATCH 076/183] Refactor --- pkg/account.go | 2 +- pkg/{controller.go => cli.go} | 0 pkg/collections.go | 128 --------------------------- pkg/download.go | 19 ---- pkg/export.go | 31 ------- pkg/remote_sync.go | 159 ++++++++++++++++++++++++++-------- pkg/sign_in.go | 2 +- pkg/store.go | 4 +- pkg/sync.go | 84 ++++++++++++++++++ 9 files changed, 210 insertions(+), 219 deletions(-) rename pkg/{controller.go => cli.go} (100%) delete mode 100644 pkg/collections.go delete mode 100644 pkg/export.go create mode 100644 pkg/sync.go diff --git a/pkg/account.go b/pkg/account.go index 87e145636..905134235 100644 --- a/pkg/account.go +++ b/pkg/account.go @@ -45,7 +45,7 @@ func (c *ClICtrl) AddAccount(cxt context.Context) { if verifyEmail || srpAttr.IsEmailMFAEnabled { authResponse, flowErr = c.validateEmail(cxt, email) } else { - authResponse, keyEncKey, flowErr = c.signInViaPassword(cxt, email, srpAttr) + authResponse, keyEncKey, flowErr = c.signInViaPassword(cxt, srpAttr) } if flowErr != nil { return diff --git a/pkg/controller.go b/pkg/cli.go similarity index 100% rename from pkg/controller.go rename to pkg/cli.go diff --git a/pkg/collections.go b/pkg/collections.go deleted file mode 100644 index b2828d58d..000000000 --- a/pkg/collections.go +++ /dev/null @@ -1,128 +0,0 @@ -package pkg - -import ( - "cli-go/pkg/model" - "cli-go/utils/encoding" - "context" - "encoding/json" - "fmt" - "log" - "strconv" - "time" -) - -func (c *ClICtrl) fetchRemoteCollections(ctx context.Context) error { - lastSyncTime, err2 := c.GetInt64ConfigValue(ctx, model.CollectionsSyncKey) - if err2 != nil { - return err2 - } - collections, err := c.Client.GetCollections(ctx, lastSyncTime) - if err != nil { - return fmt.Errorf("failed to get collections: %s", err) - } - maxUpdated := lastSyncTime - for _, collection := range collections { - if lastSyncTime == 0 && collection.IsDeleted { - continue - } - album, mapErr := c.mapCollectionToAlbum(ctx, collection) - if mapErr != nil { - return mapErr - } - if album.LastUpdatedAt > maxUpdated { - maxUpdated = album.LastUpdatedAt - } - albumJson := encoding.MustMarshalJSON(album) - putErr := c.PutValue(ctx, model.RemoteAlbums, []byte(strconv.FormatInt(album.ID, 10)), albumJson) - if putErr != nil { - return putErr - } - } - if maxUpdated > lastSyncTime { - err = c.PutConfigValue(ctx, model.CollectionsSyncKey, []byte(strconv.FormatInt(maxUpdated, 10))) - if err != nil { - return fmt.Errorf("failed to update last sync time: %s", err) - } - } - return nil -} - -func (c *ClICtrl) fetchRemoteFiles(ctx context.Context) error { - albums, err := c.getRemoteAlbums(ctx) - if err != nil { - return err - } - for _, album := range albums { - if album.IsDeleted { - log.Printf("Skipping album %s as it is deleted", album.AlbumName) - continue - } - lastSyncTime, lastSyncTimeErr := c.GetInt64ConfigValue(ctx, fmt.Sprintf(model.CollectionsFileSyncKeyFmt, album.ID)) - if lastSyncTimeErr != nil { - return lastSyncTimeErr - } - isFirstSync := lastSyncTime == 0 - for { - - if lastSyncTime == album.LastUpdatedAt { - break - } - if !isFirstSync { - t := time.UnixMicro(lastSyncTime) - log.Printf("Fetching files for album %s from %v\n", album.AlbumName, t) - } - files, hasMore, err := c.Client.GetFiles(ctx, album.ID, lastSyncTime) - if err != nil { - return err - } - maxUpdated := lastSyncTime - for _, file := range files { - if file.UpdationTime > maxUpdated { - maxUpdated = file.UpdationTime - } - if isFirstSync && file.IsDeleted { - // on first sync, no need to sync delete markers - continue - } - photoFile, err := c.mapApiFileToPhotoFile(ctx, album, file) - if err != nil { - return err - } - fileJson := encoding.MustMarshalJSON(photoFile) - putErr := c.PutValue(ctx, model.RemoteFiles, []byte(strconv.FormatInt(file.ID, 10)), fileJson) - if putErr != nil { - return putErr - } - } - if !hasMore { - maxUpdated = album.LastUpdatedAt - } - if maxUpdated > lastSyncTime || !hasMore { - err = c.PutConfigValue(ctx, fmt.Sprintf(model.CollectionsFileSyncKeyFmt, album.ID), []byte(strconv.FormatInt(maxUpdated, 10))) - if err != nil { - return fmt.Errorf("failed to update last sync time: %s", err) - } else { - lastSyncTime = maxUpdated - } - } - } - } - return nil -} - -func (c *ClICtrl) getRemoteAlbums(ctx context.Context) ([]model.RemoteAlbum, error) { - albums := make([]model.RemoteAlbum, 0) - albumBytes, err := c.GetAllValues(ctx, model.RemoteAlbums) - if err != nil { - return nil, err - } - for _, albumJson := range albumBytes { - album := model.RemoteAlbum{} - err = json.Unmarshal(albumJson, &album) - if err != nil { - return nil, err - } - albums = append(albums, album) - } - return albums, nil -} diff --git a/pkg/download.go b/pkg/download.go index 4e82ecdeb..22b0feab3 100644 --- a/pkg/download.go +++ b/pkg/download.go @@ -1,9 +1,7 @@ package pkg import ( - "cli-go/pkg/model" "context" - "encoding/json" "fmt" "log" "os" @@ -28,20 +26,3 @@ func (c *ClICtrl) initiateDownload(ctx context.Context) error { } return nil } - -func (c *ClICtrl) getRemoteFiles(ctx context.Context) ([]model.RemoteFile, error) { - files := make([]model.RemoteFile, 0) - fileBytes, err := c.GetAllValues(ctx, model.RemoteFiles) - if err != nil { - return nil, err - } - for _, fileJson := range fileBytes { - file := model.RemoteFile{} - err = json.Unmarshal(fileJson, &file) - if err != nil { - return nil, err - } - files = append(files, file) - } - return files, nil -} diff --git a/pkg/export.go b/pkg/export.go deleted file mode 100644 index 194515bbf..000000000 --- a/pkg/export.go +++ /dev/null @@ -1,31 +0,0 @@ -package pkg - -import ( - "context" - "fmt" - "log" -) - -func (c *ClICtrl) StartSync() error { - accounts, err := c.GetAccounts(context.Background()) - if err != nil { - return err - } - if len(accounts) == 0 { - fmt.Printf("No accounts to sync\n") - return nil - } - for _, account := range accounts { - log.SetPrefix(fmt.Sprintf("[%s-%s] ", account.App, account.Email)) - log.Println("start sync") - err = c.SyncAccount(account) - if err != nil { - fmt.Printf("Error syncing account %s: %s\n", account.Email, err) - return err - } else { - log.Println("sync done") - } - - } - return nil -} diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index f2e3c5c8c..6d6342bd7 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -2,59 +2,144 @@ package pkg import ( "cli-go/pkg/model" + "cli-go/utils/encoding" "context" - "encoding/base64" + "encoding/json" "fmt" - bolt "go.etcd.io/bbolt" "log" + "strconv" + "time" ) -func (c *ClICtrl) SyncAccount(account model.Account) error { - secretInfo, err := c.KeyHolder.LoadSecrets(account, c.CliKey) - if err != nil { - return err +func (c *ClICtrl) fetchRemoteCollections(ctx context.Context) error { + lastSyncTime, err2 := c.GetInt64ConfigValue(ctx, model.CollectionsSyncKey) + if err2 != nil { + return err2 } - ctx := c.buildRequestContext(context.Background(), account) - err = createDataBuckets(c.DB, account) + collections, err := c.Client.GetCollections(ctx, lastSyncTime) if err != nil { - return err + return fmt.Errorf("failed to get collections: %s", err) } - c.Client.AddToken(account.AccountKey(), base64.URLEncoding.EncodeToString(secretInfo.Token)) - err = c.fetchRemoteCollections(ctx) - if err != nil { - log.Printf("Error fetching collections: %s", err) + maxUpdated := lastSyncTime + for _, collection := range collections { + if lastSyncTime == 0 && collection.IsDeleted { + continue + } + album, mapErr := c.mapCollectionToAlbum(ctx, collection) + if mapErr != nil { + return mapErr + } + if album.LastUpdatedAt > maxUpdated { + maxUpdated = album.LastUpdatedAt + } + albumJson := encoding.MustMarshalJSON(album) + putErr := c.PutValue(ctx, model.RemoteAlbums, []byte(strconv.FormatInt(album.ID, 10)), albumJson) + if putErr != nil { + return putErr + } } - err = c.fetchRemoteFiles(ctx) - if err != nil { - log.Printf("Error fetching files: %s", err) - } - downloadErr := c.initiateDownload(ctx) - if downloadErr != nil { - log.Printf("Error downloading files: %s", downloadErr) - return downloadErr + if maxUpdated > lastSyncTime { + err = c.PutConfigValue(ctx, model.CollectionsSyncKey, []byte(strconv.FormatInt(maxUpdated, 10))) + if err != nil { + return fmt.Errorf("failed to update last sync time: %s", err) + } } return nil } -func (c *ClICtrl) buildRequestContext(ctx context.Context, account model.Account) context.Context { - ctx = context.WithValue(ctx, "app", string(account.App)) - ctx = context.WithValue(ctx, "account_id", account.AccountKey()) - ctx = context.WithValue(ctx, "user_id", account.UserID) - return ctx -} - -func createDataBuckets(db *bolt.DB, account model.Account) error { - return db.Update(func(tx *bolt.Tx) error { - dataBucket, err := tx.CreateBucketIfNotExists([]byte(account.AccountKey())) - if err != nil { - return fmt.Errorf("create bucket: %s", err) +func (c *ClICtrl) fetchRemoteFiles(ctx context.Context) error { + albums, err := c.getRemoteAlbums(ctx) + if err != nil { + return err + } + for _, album := range albums { + if album.IsDeleted { + log.Printf("Skipping album %s as it is deleted", album.AlbumName) + continue } - for _, subBucket := range []model.PhotosStore{model.KVConfig, model.RemoteAlbums, model.RemoteFiles} { - _, err := dataBucket.CreateBucketIfNotExists([]byte(subBucket)) + lastSyncTime, lastSyncTimeErr := c.GetInt64ConfigValue(ctx, fmt.Sprintf(model.CollectionsFileSyncKeyFmt, album.ID)) + if lastSyncTimeErr != nil { + return lastSyncTimeErr + } + isFirstSync := lastSyncTime == 0 + for { + + if lastSyncTime == album.LastUpdatedAt { + break + } + if !isFirstSync { + t := time.UnixMicro(lastSyncTime) + log.Printf("Fetching files for album %s from %v\n", album.AlbumName, t) + } + files, hasMore, err := c.Client.GetFiles(ctx, album.ID, lastSyncTime) if err != nil { return err } + maxUpdated := lastSyncTime + for _, file := range files { + if file.UpdationTime > maxUpdated { + maxUpdated = file.UpdationTime + } + if isFirstSync && file.IsDeleted { + // on first sync, no need to sync delete markers + continue + } + photoFile, err := c.mapApiFileToPhotoFile(ctx, album, file) + if err != nil { + return err + } + fileJson := encoding.MustMarshalJSON(photoFile) + putErr := c.PutValue(ctx, model.RemoteFiles, []byte(strconv.FormatInt(file.ID, 10)), fileJson) + if putErr != nil { + return putErr + } + } + if !hasMore { + maxUpdated = album.LastUpdatedAt + } + if maxUpdated > lastSyncTime || !hasMore { + err = c.PutConfigValue(ctx, fmt.Sprintf(model.CollectionsFileSyncKeyFmt, album.ID), []byte(strconv.FormatInt(maxUpdated, 10))) + if err != nil { + return fmt.Errorf("failed to update last sync time: %s", err) + } else { + lastSyncTime = maxUpdated + } + } } - return nil - }) + } + return nil +} + +func (c *ClICtrl) getRemoteAlbums(ctx context.Context) ([]model.RemoteAlbum, error) { + albums := make([]model.RemoteAlbum, 0) + albumBytes, err := c.GetAllValues(ctx, model.RemoteAlbums) + if err != nil { + return nil, err + } + for _, albumJson := range albumBytes { + album := model.RemoteAlbum{} + err = json.Unmarshal(albumJson, &album) + if err != nil { + return nil, err + } + albums = append(albums, album) + } + return albums, nil +} + +func (c *ClICtrl) getRemoteFiles(ctx context.Context) ([]model.RemoteFile, error) { + files := make([]model.RemoteFile, 0) + fileBytes, err := c.GetAllValues(ctx, model.RemoteFiles) + if err != nil { + return nil, err + } + for _, fileJson := range fileBytes { + file := model.RemoteFile{} + err = json.Unmarshal(fileJson, &file) + if err != nil { + return nil, err + } + files = append(files, file) + } + return files, nil } diff --git a/pkg/sign_in.go b/pkg/sign_in.go index ed26403ef..d8b21f951 100644 --- a/pkg/sign_in.go +++ b/pkg/sign_in.go @@ -13,7 +13,7 @@ import ( "github.com/kong/go-srp" ) -func (c *ClICtrl) signInViaPassword(ctx context.Context, email string, srpAttr *api.SRPAttributes) (*api.AuthorizationResponse, []byte, error) { +func (c *ClICtrl) signInViaPassword(ctx context.Context, srpAttr *api.SRPAttributes) (*api.AuthorizationResponse, []byte, error) { for { // CLI prompt for password password, flowErr := internal.GetSensitiveField("Enter password") diff --git a/pkg/store.go b/pkg/store.go index 6ed09697b..4bfdc8675 100644 --- a/pkg/store.go +++ b/pkg/store.go @@ -20,7 +20,7 @@ func GetDB(path string) (*bolt.DB, error) { } func (c *ClICtrl) GetInt64ConfigValue(ctx context.Context, key string) (int64, error) { - value, err := c.GetConfigValue(ctx, key) + value, err := c.getConfigValue(ctx, key) if err != nil { return 0, err } @@ -34,7 +34,7 @@ func (c *ClICtrl) GetInt64ConfigValue(ctx context.Context, key string) (int64, e return result, nil } -func (c *ClICtrl) GetConfigValue(ctx context.Context, key string) ([]byte, error) { +func (c *ClICtrl) getConfigValue(ctx context.Context, key string) ([]byte, error) { var value []byte err := c.DB.View(func(tx *bolt.Tx) error { kvBucket, err := getAccountStore(ctx, tx, model.KVConfig) diff --git a/pkg/sync.go b/pkg/sync.go new file mode 100644 index 000000000..e7742d29c --- /dev/null +++ b/pkg/sync.go @@ -0,0 +1,84 @@ +package pkg + +import ( + "cli-go/pkg/model" + "context" + "encoding/base64" + "fmt" + bolt "go.etcd.io/bbolt" + "log" +) + +func (c *ClICtrl) StartSync() error { + accounts, err := c.GetAccounts(context.Background()) + if err != nil { + return err + } + if len(accounts) == 0 { + fmt.Printf("No accounts to sync\n") + return nil + } + for _, account := range accounts { + log.SetPrefix(fmt.Sprintf("[%s-%s] ", account.App, account.Email)) + log.Println("start sync") + err = c.SyncAccount(account) + if err != nil { + fmt.Printf("Error syncing account %s: %s\n", account.Email, err) + return err + } else { + log.Println("sync done") + } + + } + return nil +} + +func (c *ClICtrl) SyncAccount(account model.Account) error { + secretInfo, err := c.KeyHolder.LoadSecrets(account, c.CliKey) + if err != nil { + return err + } + ctx := c.buildRequestContext(context.Background(), account) + err = createDataBuckets(c.DB, account) + if err != nil { + return err + } + c.Client.AddToken(account.AccountKey(), base64.URLEncoding.EncodeToString(secretInfo.Token)) + err = c.fetchRemoteCollections(ctx) + if err != nil { + log.Printf("Error fetching collections: %s", err) + } + err = c.fetchRemoteFiles(ctx) + if err != nil { + log.Printf("Error fetching files: %s", err) + } + downloadErr := c.initiateDownload(ctx) + if downloadErr != nil { + log.Printf("Error downloading files: %s", downloadErr) + return downloadErr + } + return nil +} + +func (c *ClICtrl) buildRequestContext(ctx context.Context, account model.Account) context.Context { + ctx = context.WithValue(ctx, "app", string(account.App)) + ctx = context.WithValue(ctx, "account_id", account.AccountKey()) + ctx = context.WithValue(ctx, "user_id", account.UserID) + return ctx +} + +func createDataBuckets(db *bolt.DB, account model.Account) error { + return db.Update(func(tx *bolt.Tx) error { + dataBucket, err := tx.CreateBucketIfNotExists([]byte(account.AccountKey())) + if err != nil { + return fmt.Errorf("create bucket: %s", err) + } + for _, subBucket := range []model.PhotosStore{model.KVConfig, model.RemoteAlbums, model.RemoteFiles} { + _, err := dataBucket.CreateBucketIfNotExists([]byte(subBucket)) + if err != nil { + return err + } + } + return nil + }) +} From 93925b0731387f954310543b033db67e446f7141 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 27 Sep 2023 14:09:44 +0530 Subject: [PATCH 077/183] Move CLIkey in KeyHolder --- main.go | 3 +-- pkg/account.go | 7 +++---- pkg/cli.go | 6 ++---- pkg/mappers.go | 13 +++++++------ pkg/remote_sync.go | 4 ++-- pkg/secrets/key_holder.go | 14 +++++++++----- pkg/sync.go | 2 +- 7 files changed, 25 insertions(+), 24 deletions(-) diff --git a/main.go b/main.go index be12a1ef5..51a00d0cd 100644 --- a/main.go +++ b/main.go @@ -18,8 +18,7 @@ func main() { //Host: "http://localhost:8080", }), DB: db, - CliKey: secrets.GetOrCreateClISecret(), - KeyHolder: secrets.NewKeyHolder(), + KeyHolder: secrets.NewKeyHolder(secrets.GetOrCreateClISecret()), } err = ctrl.Init() if err != nil { diff --git a/pkg/account.go b/pkg/account.go index 905134235..f11bbd6a2 100644 --- a/pkg/account.go +++ b/pkg/account.go @@ -72,7 +72,6 @@ func (c *ClICtrl) AddAccount(cxt context.Context) { func (c *ClICtrl) storeAccount(_ context.Context, email string, userID int64, app api.App, secretInfo *model.AccSecretInfo) error { // get password - secret := c.CliKey err := c.DB.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucketIfNotExists([]byte(AccBucket)) if err != nil { @@ -81,9 +80,9 @@ func (c *ClICtrl) storeAccount(_ context.Context, email string, userID int64, ap accInfo := model.Account{ Email: email, UserID: userID, - MasterKey: *model.MakeEncString(secretInfo.MasterKey, secret), - SecretKey: *model.MakeEncString(secretInfo.SecretKey, secret), - Token: *model.MakeEncString(secretInfo.Token, secret), + MasterKey: *model.MakeEncString(secretInfo.MasterKey, c.KeyHolder.DeviceKey), + SecretKey: *model.MakeEncString(secretInfo.SecretKey, c.KeyHolder.DeviceKey), + Token: *model.MakeEncString(secretInfo.Token, c.KeyHolder.DeviceKey), App: app, PublicKey: encoding.EncodeBase64(secretInfo.PublicKey), } diff --git a/pkg/cli.go b/pkg/cli.go index 3d2dd2ed0..ee80660c7 100644 --- a/pkg/cli.go +++ b/pkg/cli.go @@ -8,10 +8,8 @@ import ( ) type ClICtrl struct { - Client *api.Client - DB *bolt.DB - // CliKey is the key used to encrypt/decrypt sensitive data stored in the database - CliKey []byte + Client *api.Client + DB *bolt.DB KeyHolder *secrets.KeyHolder } diff --git a/pkg/mappers.go b/pkg/mappers.go index 510cad36d..9ac99f3b3 100644 --- a/pkg/mappers.go +++ b/pkg/mappers.go @@ -4,6 +4,7 @@ import ( "cli-go/internal/api" eCrypto "cli-go/internal/crypto" "cli-go/pkg/model" + "cli-go/pkg/secrets" "cli-go/utils/encoding" "context" "encoding/json" @@ -11,7 +12,7 @@ import ( "log" ) -func (c *ClICtrl) mapCollectionToAlbum(ctx context.Context, collection api.Collection) (*model.RemoteAlbum, error) { +func (c *ClICtrl) mapCollectionToAlbum(ctx context.Context, collection api.Collection, holder *secrets.KeyHolder) (*model.RemoteAlbum, error) { var album model.RemoteAlbum userID := ctx.Value("user_id").(int64) album.OwnerID = collection.Owner.ID @@ -19,11 +20,11 @@ func (c *ClICtrl) mapCollectionToAlbum(ctx context.Context, collection api.Colle album.IsShared = collection.Owner.ID != userID album.LastUpdatedAt = collection.UpdationTime album.IsDeleted = collection.IsDeleted - collectionKey, err := c.KeyHolder.GetCollectionKey(ctx, collection) + collectionKey, err := holder.GetCollectionKey(ctx, collection) if err != nil { return nil, err } - album.AlbumKey = *model.MakeEncString(collectionKey, c.CliKey) + album.AlbumKey = *model.MakeEncString(collectionKey, holder.DeviceKey) var name string if collection.EncryptedName != "" { decrName, err := eCrypto.SecretBoxOpenBase64(collection.EncryptedName, collection.NameDecryptionNonce, collectionKey) @@ -69,11 +70,11 @@ func (c *ClICtrl) mapCollectionToAlbum(ctx context.Context, collection api.Colle return &album, nil } -func (c *ClICtrl) mapApiFileToPhotoFile(ctx context.Context, album model.RemoteAlbum, file api.File) (*model.RemoteFile, error) { +func (c *ClICtrl) mapApiFileToPhotoFile(ctx context.Context, album model.RemoteAlbum, file api.File, holder *secrets.KeyHolder) (*model.RemoteFile, error) { if file.IsDeleted { return nil, errors.New("file is deleted") } - albumKey := album.AlbumKey.MustDecrypt(c.CliKey) + albumKey := album.AlbumKey.MustDecrypt(holder.DeviceKey) fileKey, err := eCrypto.SecretBoxOpen( encoding.DecodeBase64(file.EncryptedKey), encoding.DecodeBase64(file.KeyDecryptionNonce), @@ -84,7 +85,7 @@ func (c *ClICtrl) mapApiFileToPhotoFile(ctx context.Context, album model.RemoteA var photoFile model.RemoteFile photoFile.ID = file.ID photoFile.LastUpdateTime = file.UpdationTime - photoFile.Key = *model.MakeEncString(fileKey, c.CliKey) + photoFile.Key = *model.MakeEncString(fileKey, holder.DeviceKey) photoFile.FileNonce = file.File.DecryptionHeader photoFile.ThumbnailNonce = file.Thumbnail.DecryptionHeader photoFile.OwnerID = file.OwnerID diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index 6d6342bd7..fa0dc6828 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -25,7 +25,7 @@ func (c *ClICtrl) fetchRemoteCollections(ctx context.Context) error { if lastSyncTime == 0 && collection.IsDeleted { continue } - album, mapErr := c.mapCollectionToAlbum(ctx, collection) + album, mapErr := c.mapCollectionToAlbum(ctx, collection, c.KeyHolder) if mapErr != nil { return mapErr } @@ -84,7 +84,7 @@ func (c *ClICtrl) fetchRemoteFiles(ctx context.Context) error { // on first sync, no need to sync delete markers continue } - photoFile, err := c.mapApiFileToPhotoFile(ctx, album, file) + photoFile, err := c.mapApiFileToPhotoFile(ctx, album, file, c.KeyHolder) if err != nil { return err } diff --git a/pkg/secrets/key_holder.go b/pkg/secrets/key_holder.go index 336e3178f..621077b1f 100644 --- a/pkg/secrets/key_holder.go +++ b/pkg/secrets/key_holder.go @@ -10,14 +10,18 @@ import ( ) type KeyHolder struct { + // DeviceKey is the key used to encrypt/decrypt the data while storing sensitive + // information on the disk. Usually, it should be stored in OS Keychain. + DeviceKey []byte AccountSecrets map[string]*model.AccSecretInfo CollectionKeys map[string][]byte } -func NewKeyHolder() *KeyHolder { +func NewKeyHolder(deviceKey []byte) *KeyHolder { return &KeyHolder{ AccountSecrets: make(map[string]*model.AccSecretInfo), CollectionKeys: make(map[string][]byte), + DeviceKey: deviceKey, } } @@ -25,10 +29,10 @@ func NewKeyHolder() *KeyHolder { // It decrypts the token key, master key, and secret key using the CLI key. // The decrypted keys and the decoded public key are stored in the AccountSecrets map using the account key as the map key. // It returns the account secret information or an error if the decryption fails. -func (k *KeyHolder) LoadSecrets(account model.Account, cliKey []byte) (*model.AccSecretInfo, error) { - tokenKey := account.Token.MustDecrypt(cliKey) - masterKey := account.MasterKey.MustDecrypt(cliKey) - secretKey := account.SecretKey.MustDecrypt(cliKey) +func (k *KeyHolder) LoadSecrets(account model.Account) (*model.AccSecretInfo, error) { + tokenKey := account.Token.MustDecrypt(k.DeviceKey) + masterKey := account.MasterKey.MustDecrypt(k.DeviceKey) + secretKey := account.SecretKey.MustDecrypt(k.DeviceKey) k.AccountSecrets[account.AccountKey()] = &model.AccSecretInfo{ Token: tokenKey, MasterKey: masterKey, diff --git a/pkg/sync.go b/pkg/sync.go index e7742d29c..dd5680245 100644 --- a/pkg/sync.go +++ b/pkg/sync.go @@ -34,7 +34,7 @@ func (c *ClICtrl) StartSync() error { } func (c *ClICtrl) SyncAccount(account model.Account) error { - secretInfo, err := c.KeyHolder.LoadSecrets(account, c.CliKey) + secretInfo, err := c.KeyHolder.LoadSecrets(account) if err != nil { return err } From 342325a56aa152a22d05498410d0e598e307eeb9 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 27 Sep 2023 14:13:42 +0530 Subject: [PATCH 078/183] Move mappers inside separate pkg --- pkg/{mappers.go => mapper/photo.go} | 6 +++--- pkg/remote_sync.go | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) rename pkg/{mappers.go => mapper/photo.go} (92%) diff --git a/pkg/mappers.go b/pkg/mapper/photo.go similarity index 92% rename from pkg/mappers.go rename to pkg/mapper/photo.go index 9ac99f3b3..63617c5ad 100644 --- a/pkg/mappers.go +++ b/pkg/mapper/photo.go @@ -1,4 +1,4 @@ -package pkg +package mapper import ( "cli-go/internal/api" @@ -12,7 +12,7 @@ import ( "log" ) -func (c *ClICtrl) mapCollectionToAlbum(ctx context.Context, collection api.Collection, holder *secrets.KeyHolder) (*model.RemoteAlbum, error) { +func MapCollectionToAlbum(ctx context.Context, collection api.Collection, holder *secrets.KeyHolder) (*model.RemoteAlbum, error) { var album model.RemoteAlbum userID := ctx.Value("user_id").(int64) album.OwnerID = collection.Owner.ID @@ -70,7 +70,7 @@ func (c *ClICtrl) mapCollectionToAlbum(ctx context.Context, collection api.Colle return &album, nil } -func (c *ClICtrl) mapApiFileToPhotoFile(ctx context.Context, album model.RemoteAlbum, file api.File, holder *secrets.KeyHolder) (*model.RemoteFile, error) { +func MapApiFileToPhotoFile(ctx context.Context, album model.RemoteAlbum, file api.File, holder *secrets.KeyHolder) (*model.RemoteFile, error) { if file.IsDeleted { return nil, errors.New("file is deleted") } diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index fa0dc6828..78db91450 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -1,6 +1,7 @@ package pkg import ( + "cli-go/pkg/mapper" "cli-go/pkg/model" "cli-go/utils/encoding" "context" @@ -25,7 +26,7 @@ func (c *ClICtrl) fetchRemoteCollections(ctx context.Context) error { if lastSyncTime == 0 && collection.IsDeleted { continue } - album, mapErr := c.mapCollectionToAlbum(ctx, collection, c.KeyHolder) + album, mapErr := mapper.MapCollectionToAlbum(ctx, collection, c.KeyHolder) if mapErr != nil { return mapErr } @@ -84,7 +85,7 @@ func (c *ClICtrl) fetchRemoteFiles(ctx context.Context) error { // on first sync, no need to sync delete markers continue } - photoFile, err := c.mapApiFileToPhotoFile(ctx, album, file, c.KeyHolder) + photoFile, err := mapper.MapApiFileToPhotoFile(ctx, album, file, c.KeyHolder) if err != nil { return err } From ece4d1f7193b13aa5ab9dc47517954bda2ac440b Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 4 Oct 2023 08:31:24 +0530 Subject: [PATCH 079/183] Rename --- internal/api/client.go | 6 +++--- pkg/secrets/key_holder.go | 2 +- pkg/store.go | 4 ++-- pkg/sync.go | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/api/client.go b/internal/api/client.go index 38756f7f9..f392fc5a2 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -72,9 +72,9 @@ func NewClient(p Params) *Client { } func attachToken(req *resty.Request) { - accountId := readValueFromContext(req.Context(), "account_id") - if accountId != nil && accountId != "" { - if token, ok := tokenMap[accountId.(string)]; ok { + accountKey := readValueFromContext(req.Context(), "account_key") + if accountKey != nil && accountKey != "" { + if token, ok := tokenMap[accountKey.(string)]; ok { req.SetHeader(TokenHeader, token) } } diff --git a/pkg/secrets/key_holder.go b/pkg/secrets/key_holder.go index 621077b1f..dc82c72d1 100644 --- a/pkg/secrets/key_holder.go +++ b/pkg/secrets/key_holder.go @@ -43,7 +43,7 @@ func (k *KeyHolder) LoadSecrets(account model.Account) (*model.AccSecretInfo, er } func (k *KeyHolder) GetAccountSecretInfo(ctx context.Context) *model.AccSecretInfo { - accountKey := ctx.Value("account_id").(string) + accountKey := ctx.Value("account_key").(string) return k.AccountSecrets[accountKey] } diff --git a/pkg/store.go b/pkg/store.go index 4bfdc8675..7dd732198 100644 --- a/pkg/store.go +++ b/pkg/store.go @@ -82,8 +82,8 @@ func (c *ClICtrl) PutValue(ctx context.Context, store model.PhotosStore, key []b }) } func getAccountStore(ctx context.Context, tx *bolt.Tx, storeType model.PhotosStore) (*bolt.Bucket, error) { - accountId := ctx.Value("account_id").(string) - accountBucket := tx.Bucket([]byte(accountId)) + accountKey := ctx.Value("account_key").(string) + accountBucket := tx.Bucket([]byte(accountKey)) if accountBucket == nil { return nil, fmt.Errorf("account bucket not found") } diff --git a/pkg/sync.go b/pkg/sync.go index dd5680245..02f5a068c 100644 --- a/pkg/sync.go +++ b/pkg/sync.go @@ -62,7 +62,7 @@ func (c *ClICtrl) SyncAccount(account model.Account) error { func (c *ClICtrl) buildRequestContext(ctx context.Context, account model.Account) context.Context { ctx = context.WithValue(ctx, "app", string(account.App)) - ctx = context.WithValue(ctx, "account_id", account.AccountKey()) + ctx = context.WithValue(ctx, "account_key", account.AccountKey()) ctx = context.WithValue(ctx, "user_id", account.UserID) return ctx } From 5535f5bc5c05587095d257c2d17c71a2467e83d7 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 4 Oct 2023 11:50:52 +0530 Subject: [PATCH 080/183] Add logic to create photos folder in hw and create album folders --- pkg/remote_to_local_sync.go | 43 +++++++++++++++++++++++++++++++++++++ pkg/sync.go | 5 +++++ 2 files changed, 48 insertions(+) create mode 100644 pkg/remote_to_local_sync.go diff --git a/pkg/remote_to_local_sync.go b/pkg/remote_to_local_sync.go new file mode 100644 index 000000000..e9ace63a3 --- /dev/null +++ b/pkg/remote_to_local_sync.go @@ -0,0 +1,43 @@ +package pkg + +import ( + "context" + "fmt" + "log" + "os" +) + +// CreateLocalFolderForRemoteAlbums will get all the remote albums and create a local folder for each of them +func (c *ClICtrl) CreateLocalFolderForRemoteAlbums(ctx context.Context) error { + homeDir, err := os.UserHomeDir() + if err != nil { + return err + } + path := fmt.Sprintf("%s/%s", homeDir, "photos") + if _, err := os.Stat(path); os.IsNotExist(err) { + err = os.Mkdir(path, 0755) + if err != nil { + return err + } + } + albums, err := c.getRemoteAlbums(ctx) + if err != nil { + return err + } + for _, album := range albums { + if album.IsDeleted { + continue + } + albumPath := path + "/" + album.AlbumName + // create the folder if it doesn't exist + if _, err := os.Stat(albumPath); os.IsNotExist(err) { + err = os.Mkdir(albumPath, 0755) + if err != nil { + return err + } + } else { + log.Printf("Folder %s already exists", albumPath) + } + } + return nil +} diff --git a/pkg/sync.go b/pkg/sync.go index 02f5a068c..d253be2ae 100644 --- a/pkg/sync.go +++ b/pkg/sync.go @@ -52,6 +52,11 @@ func (c *ClICtrl) SyncAccount(account model.Account) error { if err != nil { log.Printf("Error fetching files: %s", err) } + err = c.CreateLocalFolderForRemoteAlbums(ctx) + if err != nil { + log.Printf("Error creating local folders: %s", err) + return err + } downloadErr := c.initiateDownload(ctx) if downloadErr != nil { log.Printf("Error downloading files: %s", downloadErr) From c6836f6d884fee833ba13ca44fb2a794ee0f4466 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 4 Oct 2023 11:52:34 +0530 Subject: [PATCH 081/183] Add support for storing albumEntries --- pkg/model/constants.go | 7 ++++--- pkg/model/remote.go | 7 +++++++ pkg/remote_sync.go | 26 ++++++++++++++++++++++---- pkg/sync.go | 2 +- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/pkg/model/constants.go b/pkg/model/constants.go index 65db2b450..2a9f4cafa 100644 --- a/pkg/model/constants.go +++ b/pkg/model/constants.go @@ -3,9 +3,10 @@ package model type PhotosStore string const ( - KVConfig PhotosStore = "kvConfig" - RemoteAlbums PhotosStore = "remoteAlbums" - RemoteFiles PhotosStore = "remoteFiles" + KVConfig PhotosStore = "kvConfig" + RemoteAlbums PhotosStore = "remoteAlbums" + RemoteFiles PhotosStore = "remoteFiles" + RemoteAlbumEntries PhotosStore = "remoteAlbumEntries" ) const ( diff --git a/pkg/model/remote.go b/pkg/model/remote.go index 812fe5475..e3dd4e1ee 100644 --- a/pkg/model/remote.go +++ b/pkg/model/remote.go @@ -30,3 +30,10 @@ type RemoteAlbum struct { SharedMeta map[string]interface{} `json:"sharedMeta"` LastUpdatedAt int64 `json:"lastUpdatedAt"` } + +type AlbumFileEntry struct { + FileID int64 `json:"fileID"` + AlbumID int64 `json:"albumID"` + IsDeleted bool `json:"isDeleted"` + SyncedLocally bool `json:"localSync"` +} diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index 78db91450..a6d6cd481 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -58,16 +58,23 @@ func (c *ClICtrl) fetchRemoteFiles(ctx context.Context) error { log.Printf("Skipping album %s as it is deleted", album.AlbumName) continue } + lastSyncTime, lastSyncTimeErr := c.GetInt64ConfigValue(ctx, fmt.Sprintf(model.CollectionsFileSyncKeyFmt, album.ID)) if lastSyncTimeErr != nil { return lastSyncTimeErr } - isFirstSync := lastSyncTime == 0 - for { + isFirstSync := lastSyncTime == 0 + + for { if lastSyncTime == album.LastUpdatedAt { break } + if isFirstSync { + log.Printf("First sync for album %s\n", album.AlbumName) + } else { + log.Printf("Syncing album %s\n from %s", album.AlbumName, time.UnixMicro(lastSyncTime)) + } if !isFirstSync { t := time.UnixMicro(lastSyncTime) log.Printf("Fetching files for album %s from %v\n", album.AlbumName, t) @@ -85,12 +92,22 @@ func (c *ClICtrl) fetchRemoteFiles(ctx context.Context) error { // on first sync, no need to sync delete markers continue } + albumEntry := model.AlbumFileEntry{AlbumID: album.ID, FileID: file.ID, IsDeleted: file.IsDeleted, SyncedLocally: false} + albumEntryJson := encoding.MustMarshalJSON(albumEntry) + putErr := c.PutValue(ctx, model.RemoteAlbumEntries, []byte(fmt.Sprintf("%d:%d", album.ID, file.ID)), albumEntryJson) + if putErr != nil { + return putErr + } + if file.IsDeleted { + continue + } photoFile, err := mapper.MapApiFileToPhotoFile(ctx, album, file, c.KeyHolder) if err != nil { return err } fileJson := encoding.MustMarshalJSON(photoFile) - putErr := c.PutValue(ctx, model.RemoteFiles, []byte(strconv.FormatInt(file.ID, 10)), fileJson) + // todo: use batch put + putErr = c.PutValue(ctx, model.RemoteFiles, []byte(strconv.FormatInt(file.ID, 10)), fileJson) if putErr != nil { return putErr } @@ -98,7 +115,8 @@ func (c *ClICtrl) fetchRemoteFiles(ctx context.Context) error { if !hasMore { maxUpdated = album.LastUpdatedAt } - if maxUpdated > lastSyncTime || !hasMore { + if (maxUpdated > lastSyncTime) || !hasMore { + log.Printf("Updating last sync time for album %s to %s\n", album.AlbumName, time.UnixMicro(maxUpdated)) err = c.PutConfigValue(ctx, fmt.Sprintf(model.CollectionsFileSyncKeyFmt, album.ID), []byte(strconv.FormatInt(maxUpdated, 10))) if err != nil { return fmt.Errorf("failed to update last sync time: %s", err) diff --git a/pkg/sync.go b/pkg/sync.go index d253be2ae..d4a60f810 100644 --- a/pkg/sync.go +++ b/pkg/sync.go @@ -78,7 +78,7 @@ func createDataBuckets(db *bolt.DB, account model.Account) error { if err != nil { return fmt.Errorf("create bucket: %s", err) } - for _, subBucket := range []model.PhotosStore{model.KVConfig, model.RemoteAlbums, model.RemoteFiles} { + for _, subBucket := range []model.PhotosStore{model.KVConfig, model.RemoteAlbums, model.RemoteFiles, model.RemoteAlbumEntries} { _, err := dataBucket.CreateBucketIfNotExists([]byte(subBucket)) if err != nil { return err From 3ae0e758208419e1ffba094e602df706ef6494ef Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 4 Oct 2023 19:24:33 +0530 Subject: [PATCH 082/183] Handle album renames & local folder renames as well --- pkg/model/export/metadata.go | 28 ++++++ pkg/remote_to_local_sync.go | 165 +++++++++++++++++++++++++++++++---- 2 files changed, 175 insertions(+), 18 deletions(-) create mode 100644 pkg/model/export/metadata.go diff --git a/pkg/model/export/metadata.go b/pkg/model/export/metadata.go new file mode 100644 index 000000000..de48c4675 --- /dev/null +++ b/pkg/model/export/metadata.go @@ -0,0 +1,28 @@ +package export + +type AlbumMetadata struct { + ID int64 `json:"id"` + OwnerID int64 `json:"ownerID"` + AlbumName string `json:"albumName"` + IsDeleted bool `json:"isDeleted"` + // This is to handle the case where two accounts are exporting to the same directory + // and a album is shared between them + AccountOwnerIDs []int64 `json:"accountOwnerIDs"` + + // Folder name is the name of the disk folder that contains the album data + // exclude this from json serialization + FolderName string `json:"-"` +} + +// AddAccountOwner adds the given account id to the list of account owners +// if it is not already present. Returns true if the account id was added +// and false otherwise +func (a *AlbumMetadata) AddAccountOwner(id int64) bool { + for _, ownerID := range a.AccountOwnerIDs { + if ownerID == id { + return false + } + } + a.AccountOwnerIDs = append(a.AccountOwnerIDs, id) + return true +} diff --git a/pkg/remote_to_local_sync.go b/pkg/remote_to_local_sync.go index e9ace63a3..19e890b41 100644 --- a/pkg/remote_to_local_sync.go +++ b/pkg/remote_to_local_sync.go @@ -1,43 +1,172 @@ package pkg import ( + "cli-go/pkg/model/export" "context" + "encoding/json" "fmt" "log" "os" + "strings" + + "path/filepath" ) -// CreateLocalFolderForRemoteAlbums will get all the remote albums and create a local folder for each of them func (c *ClICtrl) CreateLocalFolderForRemoteAlbums(ctx context.Context) error { - homeDir, err := os.UserHomeDir() - if err != nil { - return err - } - path := fmt.Sprintf("%s/%s", homeDir, "photos") - if _, err := os.Stat(path); os.IsNotExist(err) { - err = os.Mkdir(path, 0755) - if err != nil { - return err - } + path, pathErr := exportHome(ctx) + if pathErr != nil { + return pathErr } + albums, err := c.getRemoteAlbums(ctx) if err != nil { return err } + userID := ctx.Value("user_id").(int64) + folderToMetaMap, albumIDToMetaMap, err := readFolderMetadata(path) + if err != nil { + return err + } + for _, album := range albums { if album.IsDeleted { + if meta, ok := albumIDToMetaMap[album.ID]; ok { + log.Printf("Deleting album %s as it is deleted", meta.AlbumName) + if err = os.RemoveAll(filepath.Join(path, meta.FolderName)); err != nil { + return err + } + delete(folderToMetaMap, meta.FolderName) + delete(albumIDToMetaMap, meta.ID) + } continue } - albumPath := path + "/" + album.AlbumName - // create the folder if it doesn't exist - if _, err := os.Stat(albumPath); os.IsNotExist(err) { - err = os.Mkdir(albumPath, 0755) - if err != nil { - return err + metaByID := albumIDToMetaMap[album.ID] + if metaByID != nil { + if strings.EqualFold(metaByID.AlbumName, album.AlbumName) { + //log.Printf("Skipping album %s as it already exists", album.AlbumName) + continue + } + } + + albumFolderName := album.AlbumName + albumID := album.ID + + if _, ok := folderToMetaMap[albumFolderName]; ok { + for i := 1; ; i++ { + newAlbumName := fmt.Sprintf("%s_%d", albumFolderName, i) + if _, ok := folderToMetaMap[newAlbumName]; !ok { + albumFolderName = newAlbumName + break + } + } + } + // Create album and meta folders if they don't exist + albumPath := filepath.Join(path, albumFolderName) + metaPath := filepath.Join(albumPath, ".meta") + if metaByID == nil { + log.Printf("Adding folder %s for album %s", albumFolderName, album.AlbumName) + for _, p := range []string{albumPath, metaPath} { + if _, err := os.Stat(p); os.IsNotExist(err) { + if err = os.Mkdir(p, 0755); err != nil { + return err + } + } } } else { - log.Printf("Folder %s already exists", albumPath) + // rename meta.FolderName to albumFolderName + oldAlbumPath := filepath.Join(path, metaByID.FolderName) + log.Printf("Renaming path from %s to %s for album %s", oldAlbumPath, albumPath, album.AlbumName) + if err = os.Rename(oldAlbumPath, albumPath); err != nil { + return err + } } + // Handle meta file + metaFilePath := filepath.Join(path, albumFolderName, ".meta", "album_meta.json") + metaData := export.AlbumMetadata{ + ID: album.ID, + OwnerID: album.OwnerID, + AlbumName: album.AlbumName, + IsDeleted: album.IsDeleted, + AccountOwnerIDs: []int64{userID}, + FolderName: albumFolderName, + } + if err = writeJSONToFile(metaFilePath, metaData); err != nil { + return err + } + folderToMetaMap[albumFolderName] = &metaData + albumIDToMetaMap[albumID] = &metaData } return nil } + +// readFolderMetadata returns a map of folder name to album metadata for all folders in the given path +// and a map of album ID to album metadata for all albums in the given path. +func readFolderMetadata(path string) (map[string]*export.AlbumMetadata, map[int64]*export.AlbumMetadata, error) { + result := make(map[string]*export.AlbumMetadata) + albumIdToMetadataMap := make(map[int64]*export.AlbumMetadata) + // Read the top-level directories in the given path + entries, err := os.ReadDir(path) + if err != nil { + return nil, nil, err + } + for _, entry := range entries { + if entry.IsDir() { + dirName := entry.Name() + metaFilePath := filepath.Join(path, dirName, ".meta", "album_meta.json") + // Initialize as nil, will remain nil if JSON file is not found or not readable + result[dirName] = nil + // Read the JSON file if it exists + if _, err := os.Stat(metaFilePath); err == nil { + var metaData export.AlbumMetadata + metaDataBytes, err := os.ReadFile(metaFilePath) + if err != nil { + continue // Skip this entry if reading fails + } + + if err := json.Unmarshal(metaDataBytes, &metaData); err == nil { + metaData.FolderName = dirName + result[dirName] = &metaData + albumIdToMetadataMap[metaData.ID] = &metaData + } + } + } + } + return result, albumIdToMetadataMap, nil +} + +func writeJSONToFile(filePath string, data interface{}) error { + file, err := os.Create(filePath) + if err != nil { + return err + } + defer file.Close() + + encoder := json.NewEncoder(file) + return encoder.Encode(data) +} + +func readJSONFromFile(filePath string, data interface{}) error { + file, err := os.Open(filePath) + if err != nil { + return err + } + defer file.Close() + + decoder := json.NewDecoder(file) + return decoder.Decode(data) +} + +func exportHome(ctx context.Context) (string, error) { + homeDir, err := os.UserHomeDir() + if err != nil { + return "", err + } + path := fmt.Sprintf("%s/%s", homeDir, "photos") + if _, err = os.Stat(path); os.IsNotExist(err) { + err = os.Mkdir(path, 0755) + if err != nil { + return "", err + } + } + return path, nil +} From a571116f0ee46e8762cdf153b5212ff8a798bb22 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 9 Oct 2023 20:56:35 +0530 Subject: [PATCH 083/183] Scaffold code to initiate file download --- pkg/remote_sync.go | 17 ++++++++++ pkg/remote_to_local_file_sync.go | 57 ++++++++++++++++++++++++++++++++ pkg/remote_to_local_sync.go | 2 +- pkg/store.go | 14 ++++++++ pkg/sync.go | 10 ++++-- 5 files changed, 97 insertions(+), 3 deletions(-) create mode 100644 pkg/remote_to_local_file_sync.go diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index a6d6cd481..07bb1ebd3 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -162,3 +162,20 @@ func (c *ClICtrl) getRemoteFiles(ctx context.Context) ([]model.RemoteFile, error } return files, nil } + +func (c *ClICtrl) getRemoteAlbumEntries(ctx context.Context) ([]model.AlbumFileEntry, error) { + entries := make([]model.AlbumFileEntry, 0) + entryBytes, err := c.GetAllValues(ctx, model.RemoteAlbumEntries) + if err != nil { + return nil, err + } + for _, entryJson := range entryBytes { + entry := model.AlbumFileEntry{} + err = json.Unmarshal(entryJson, &entry) + if err != nil { + return nil, err + } + entries = append(entries, entry) + } + return entries, nil +} diff --git a/pkg/remote_to_local_file_sync.go b/pkg/remote_to_local_file_sync.go new file mode 100644 index 000000000..7cd649baa --- /dev/null +++ b/pkg/remote_to_local_file_sync.go @@ -0,0 +1,57 @@ +package pkg + +import ( + "cli-go/pkg/model" + "cli-go/utils/encoding" + "context" + "encoding/json" + "fmt" + "log" +) + +func (c *ClICtrl) syncFiles(ctx context.Context) error { + home, err := exportHome(ctx) + if err != nil { + return err + } + _, albumIDToMetaMap, err := readFolderMetadata(home) + if err != nil { + return err + } + entries, err := c.getRemoteAlbumEntries(ctx) + if err != nil { + return err + } + for _, entry := range entries { + if entry.SyncedLocally { + continue + } + albumInfo, ok := albumIDToMetaMap[entry.AlbumID] + if !ok { + log.Printf("Album %d not found in local metadata", entry.AlbumID) + continue + } + if albumInfo.IsDeleted { + entry.IsDeleted = true + albumEntryJson := encoding.MustMarshalJSON(entry) + putErr := c.PutValue(ctx, model.RemoteAlbumEntries, []byte(fmt.Sprintf("%d:%d", entry.AlbumID, entry.FileID)), albumEntryJson) + if putErr != nil { + return putErr + } + continue + } + fileBytes, err := c.GetValue(ctx, model.RemoteAlbumEntries, []byte(fmt.Sprintf("%d:%d", entry.AlbumID, entry.FileID))) + if err != nil { + return err + } + if fileBytes != nil { + var existingEntry model.RemoteFile + err = json.Unmarshal(fileBytes, &existingEntry) + if err != nil { + return err + } + log.Printf("Should download %s into %s", existingEntry.Metadata["name"], albumInfo.FolderName) + } + } + return nil +} diff --git a/pkg/remote_to_local_sync.go b/pkg/remote_to_local_sync.go index 19e890b41..07c4b8c66 100644 --- a/pkg/remote_to_local_sync.go +++ b/pkg/remote_to_local_sync.go @@ -12,7 +12,7 @@ import ( "path/filepath" ) -func (c *ClICtrl) CreateLocalFolderForRemoteAlbums(ctx context.Context) error { +func (c *ClICtrl) createLocalFolderForRemoteAlbums(ctx context.Context) error { path, pathErr := exportHome(ctx) if pathErr != nil { return pathErr diff --git a/pkg/store.go b/pkg/store.go index 7dd732198..cd03c196b 100644 --- a/pkg/store.go +++ b/pkg/store.go @@ -81,6 +81,20 @@ func (c *ClICtrl) PutValue(ctx context.Context, store model.PhotosStore, key []b return kvBucket.Put(key, value) }) } + +// GetValue +func (c *ClICtrl) GetValue(ctx context.Context, store model.PhotosStore, key []byte) ([]byte, error) { + var value []byte + err := c.DB.View(func(tx *bolt.Tx) error { + kvBucket, err := getAccountStore(ctx, tx, store) + if err != nil { + return err + } + value = kvBucket.Get(key) + return nil + }) + return value, err +} func getAccountStore(ctx context.Context, tx *bolt.Tx, storeType model.PhotosStore) (*bolt.Bucket, error) { accountKey := ctx.Value("account_key").(string) accountBucket := tx.Bucket([]byte(accountKey)) diff --git a/pkg/sync.go b/pkg/sync.go index d4a60f810..f6b12dee4 100644 --- a/pkg/sync.go +++ b/pkg/sync.go @@ -5,8 +5,9 @@ import ( "context" "encoding/base64" "fmt" - bolt "go.etcd.io/bbolt" "log" + + bolt "go.etcd.io/bbolt" ) func (c *ClICtrl) StartSync() error { @@ -52,11 +53,16 @@ func (c *ClICtrl) SyncAccount(account model.Account) error { if err != nil { log.Printf("Error fetching files: %s", err) } - err = c.CreateLocalFolderForRemoteAlbums(ctx) + err = c.createLocalFolderForRemoteAlbums(ctx) if err != nil { log.Printf("Error creating local folders: %s", err) return err } + err = c.syncFiles(ctx) + if err != nil { + log.Printf("Error syncing files: %s", err) + return err + } downloadErr := c.initiateDownload(ctx) if downloadErr != nil { log.Printf("Error downloading files: %s", downloadErr) From ee1cc3ca91ba9a7bbf46955cc9090f80e5314051 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 11 Oct 2023 10:06:22 +0530 Subject: [PATCH 084/183] Add logic to map remoteFile to diskMetadata --- pkg/download.go | 18 ++--- pkg/mapper/photo.go | 18 ++++- pkg/model/export/location.go | 6 ++ pkg/model/export/metadata.go | 19 +++++ pkg/model/remote.go | 117 +++++++++++++++++++++++++++++++ pkg/remote_sync.go | 6 +- pkg/remote_to_local_file_sync.go | 12 +++- 7 files changed, 180 insertions(+), 16 deletions(-) create mode 100644 pkg/model/export/location.go diff --git a/pkg/download.go b/pkg/download.go index 22b0feab3..785e0d76e 100644 --- a/pkg/download.go +++ b/pkg/download.go @@ -3,7 +3,6 @@ package pkg import ( "context" "fmt" - "log" "os" ) @@ -16,13 +15,14 @@ func (c *ClICtrl) initiateDownload(ctx context.Context) error { if err != nil { return err } - for _, file := range files { - downloadPath := fmt.Sprintf("%s/%d", dir, file.ID) - log.Printf("Downloading file %d to %s", file.ID, downloadPath) - //err = c.Client.DownloadFile(ctx, file.ID, downloadPath) - //if err != nil { - // return err - //} - } + fmt.Println("total files to download: in diroector", len(files), dir) + //for _, file := range files { + // //downloadPath := fmt.Sprintf("%s/%d", dir, file.ID) + // //log.Printf("Downloading file %d to %s", file.ID, downloadPath) + // //err = c.Client.DownloadFile(ctx, file.ID, downloadPath) + // //if err != nil { + // // return err + // //} + //} return nil } diff --git a/pkg/mapper/photo.go b/pkg/mapper/photo.go index 63617c5ad..f35c49b99 100644 --- a/pkg/mapper/photo.go +++ b/pkg/mapper/photo.go @@ -4,6 +4,7 @@ import ( "cli-go/internal/api" eCrypto "cli-go/internal/crypto" "cli-go/pkg/model" + "cli-go/pkg/model/export" "cli-go/pkg/secrets" "cli-go/utils/encoding" "context" @@ -126,5 +127,20 @@ func MapApiFileToPhotoFile(ctx context.Context, album model.RemoteAlbum, file ap } } return &photoFile, nil - +} + +func MapRemoteFileToDiskMetadata(file model.RemoteFile) export.DiskFileMetadata { + result := export.DiskFileMetadata{ + Title: file.GetTitle(), + Description: file.GetCaption(), + CreationTime: file.GetCreationTime(), + ModificationTime: file.GetModificationTime(), + Location: file.GetLatlong(), + Info: &export.Info{ + ID: file.ID, + Hash: file.GetFileHash(), + OwnerID: file.OwnerID, + }, + } + return result } diff --git a/pkg/model/export/location.go b/pkg/model/export/location.go new file mode 100644 index 000000000..5dff62c26 --- /dev/null +++ b/pkg/model/export/location.go @@ -0,0 +1,6 @@ +package export + +type Location struct { + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` +} diff --git a/pkg/model/export/metadata.go b/pkg/model/export/metadata.go index de48c4675..e7ce17bd4 100644 --- a/pkg/model/export/metadata.go +++ b/pkg/model/export/metadata.go @@ -1,5 +1,7 @@ package export +import "time" + type AlbumMetadata struct { ID int64 `json:"id"` OwnerID int64 `json:"ownerID"` @@ -26,3 +28,20 @@ func (a *AlbumMetadata) AddAccountOwner(id int64) bool { a.AccountOwnerIDs = append(a.AccountOwnerIDs, id) return true } + +// DiskFileMetadata is the metadata for a file when exported to disk +// For S3 compliant storage, we will introduce a new struct that will contain references to the albums +type DiskFileMetadata struct { + Title string `json:"title"` + Description *string `json:"description"` + Location *Location `json:"location"` + CreationTime time.Time `json:"creationTime"` + ModificationTime time.Time `json:"modificationTime"` + Info *Info `json:"info"` +} + +type Info struct { + ID int64 `json:"id"` + Hash *string `json:"hash"` + OwnerID int64 `json:"ownerID"` +} diff --git a/pkg/model/remote.go b/pkg/model/remote.go index e3dd4e1ee..52f97104e 100644 --- a/pkg/model/remote.go +++ b/pkg/model/remote.go @@ -1,5 +1,20 @@ package model +import ( + "cli-go/pkg/model/export" + "fmt" + "time" +) + +type FileType int8 + +const ( + Image FileType = iota + Video + LivePhoto + Unknown = 127 +) + type RemoteFile struct { ID int64 `json:"id"` OwnerID int64 `json:"ownerID"` @@ -37,3 +52,105 @@ type AlbumFileEntry struct { IsDeleted bool `json:"isDeleted"` SyncedLocally bool `json:"localSync"` } + +func (r *RemoteFile) GetFileType() FileType { + value, ok := r.Metadata["fileType"] + if !ok { + panic("fileType not found in metadata") + } + switch value.(int8) { + case 0: + return Image + case 1: + return Video + case 2: + return LivePhoto + } + panic(fmt.Sprintf("invalid fileType %d", value.(int8))) +} + +func (r *RemoteFile) GetFileHash() *string { + value, ok := r.Metadata["hash"] + if !ok { + return nil + } + if str, ok := value.(string); ok { + return &str + } + return nil +} + +func (r *RemoteFile) GetTitle() string { + if r.PublicMetadata != nil { + if value, ok := r.PublicMetadata["editedName"]; ok { + return value.(string) + } + } + value, ok := r.Metadata["title"] + if !ok { + panic("title not found in metadata") + } + return value.(string) +} + +func (r *RemoteFile) GetCaption() *string { + if r.PublicMetadata != nil { + if value, ok := r.PublicMetadata["caption"]; ok { + if str, ok := value.(string); ok { + return &str + } + } + } + return nil +} + +func (r *RemoteFile) GetCreationTime() time.Time { + + if r.PublicMetadata != nil { + if value, ok := r.PublicMetadata["editedTime"]; ok && value.(float64) != 0 { + return time.UnixMicro(int64(value.(float64))) + } + } + value, ok := r.Metadata["creationTime"] + if !ok { + panic("creationTime not found in metadata") + } + return time.UnixMicro(int64(value.(float64))) +} + +func (r *RemoteFile) GetModificationTime() time.Time { + value, ok := r.Metadata["modificationTime"] + if !ok { + panic("creationTime not found in metadata") + } + return time.UnixMicro(int64(value.(float64))) +} + +func (r *RemoteFile) GetLatlong() *export.Location { + if r.ID == 10698020 { + fmt.Println("found 10698020") + } + if r.PublicMetadata != nil { + // check if lat and long key exists + if lat, ok := r.PublicMetadata["lat"]; ok { + if long, ok := r.PublicMetadata["long"]; ok { + if lat.(float64) == 0 && long.(float64) == 0 { + return nil + } + return &export.Location{ + Latitude: lat.(float64), + Longitude: long.(float64), + } + } + } + } + if lat, ok := r.Metadata["latitude"]; ok && lat != nil { + if long, ok2 := r.Metadata["longitude"]; ok2 && long != nil { + return &export.Location{ + Latitude: lat.(float64), + Longitude: long.(float64), + } + } + } + return nil +} diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index 07bb1ebd3..6b3fb54c7 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -163,14 +163,14 @@ func (c *ClICtrl) getRemoteFiles(ctx context.Context) ([]model.RemoteFile, error return files, nil } -func (c *ClICtrl) getRemoteAlbumEntries(ctx context.Context) ([]model.AlbumFileEntry, error) { - entries := make([]model.AlbumFileEntry, 0) +func (c *ClICtrl) getRemoteAlbumEntries(ctx context.Context) ([]*model.AlbumFileEntry, error) { + entries := make([]*model.AlbumFileEntry, 0) entryBytes, err := c.GetAllValues(ctx, model.RemoteAlbumEntries) if err != nil { return nil, err } for _, entryJson := range entryBytes { - entry := model.AlbumFileEntry{} + entry := &model.AlbumFileEntry{} err = json.Unmarshal(entryJson, &entry) if err != nil { return nil, err diff --git a/pkg/remote_to_local_file_sync.go b/pkg/remote_to_local_file_sync.go index 7cd649baa..8bb820eea 100644 --- a/pkg/remote_to_local_file_sync.go +++ b/pkg/remote_to_local_file_sync.go @@ -1,6 +1,7 @@ package pkg import ( + "cli-go/pkg/mapper" "cli-go/pkg/model" "cli-go/utils/encoding" "context" @@ -10,6 +11,7 @@ import ( ) func (c *ClICtrl) syncFiles(ctx context.Context) error { + log.Printf("Starting sync files") home, err := exportHome(ctx) if err != nil { return err @@ -22,6 +24,7 @@ func (c *ClICtrl) syncFiles(ctx context.Context) error { if err != nil { return err } + log.Println("total entries", len(entries)) for _, entry := range entries { if entry.SyncedLocally { continue @@ -40,17 +43,20 @@ func (c *ClICtrl) syncFiles(ctx context.Context) error { } continue } - fileBytes, err := c.GetValue(ctx, model.RemoteAlbumEntries, []byte(fmt.Sprintf("%d:%d", entry.AlbumID, entry.FileID))) + fileBytes, err := c.GetValue(ctx, model.RemoteFiles, []byte(fmt.Sprintf("%d", entry.FileID))) if err != nil { return err } if fileBytes != nil { - var existingEntry model.RemoteFile + var existingEntry *model.RemoteFile err = json.Unmarshal(fileBytes, &existingEntry) if err != nil { return err } - log.Printf("Should download %s into %s", existingEntry.Metadata["name"], albumInfo.FolderName) + fileDiskMetadata := mapper.MapRemoteFileToDiskMetadata(*existingEntry) + log.Printf("File %+v\\n ", fileDiskMetadata) + } else { + log.Fatalf("remoteFile %d not found in remoteFiles", entry.FileID) } } return nil From a2f1edb14b54c7f67e76a6ee5ccd7136bc7416c6 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 11 Oct 2023 16:57:17 +0530 Subject: [PATCH 085/183] Sort AlbumFileEntry to process non-deleted entries first --- pkg/model/remote.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/model/remote.go b/pkg/model/remote.go index 52f97104e..ed7326882 100644 --- a/pkg/model/remote.go +++ b/pkg/model/remote.go @@ -3,6 +3,7 @@ package model import ( "cli-go/pkg/model/export" "fmt" + "sort" "time" ) @@ -53,6 +54,16 @@ type AlbumFileEntry struct { SyncedLocally bool `json:"localSync"` } +// Sort list of AlbumFileEntry by IsDeleted and then by albumID +func SortAlbumFileEntry(entries []*AlbumFileEntry) { + sort.Slice(entries, func(i, j int) bool { + if entries[i].IsDeleted != entries[j].IsDeleted { + return !entries[i].IsDeleted && entries[j].IsDeleted + } + return entries[i].AlbumID < entries[j].AlbumID + }) +} + func (r *RemoteFile) GetFileType() FileType { value, ok := r.Metadata["fileType"] if !ok { From fc5db108e5da641df7b63dc1b33decc706bb209d Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 11 Oct 2023 20:13:42 +0530 Subject: [PATCH 086/183] Format json --- pkg/{remote_to_local_sync.go => remote_to_disk_album.go} | 1 + 1 file changed, 1 insertion(+) rename pkg/{remote_to_local_sync.go => remote_to_disk_album.go} (99%) diff --git a/pkg/remote_to_local_sync.go b/pkg/remote_to_disk_album.go similarity index 99% rename from pkg/remote_to_local_sync.go rename to pkg/remote_to_disk_album.go index 07c4b8c66..fdd8f28aa 100644 --- a/pkg/remote_to_local_sync.go +++ b/pkg/remote_to_disk_album.go @@ -142,6 +142,7 @@ func writeJSONToFile(filePath string, data interface{}) error { defer file.Close() encoder := json.NewEncoder(file) + encoder.SetIndent("", " ") return encoder.Encode(data) } From d3f0259f5a255c0bf85a10402f1ad90580c858e4 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 11 Oct 2023 20:14:08 +0530 Subject: [PATCH 087/183] Fix doc --- pkg/model/remote.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/model/remote.go b/pkg/model/remote.go index ed7326882..3fb0be9e4 100644 --- a/pkg/model/remote.go +++ b/pkg/model/remote.go @@ -54,7 +54,7 @@ type AlbumFileEntry struct { SyncedLocally bool `json:"localSync"` } -// Sort list of AlbumFileEntry by IsDeleted and then by albumID +// SortAlbumFileEntry sorts the given entries by isDeleted and then by albumID func SortAlbumFileEntry(entries []*AlbumFileEntry) { sort.Slice(entries, func(i, j int) bool { if entries[i].IsDeleted != entries[j].IsDeleted { From a2df3c36d5de37ed7b51336056d8a0b1d0c37feb Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 16 Oct 2023 12:52:42 +0530 Subject: [PATCH 088/183] Add method to clean up store value --- pkg/store.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/store.go b/pkg/store.go index cd03c196b..18d67547e 100644 --- a/pkg/store.go +++ b/pkg/store.go @@ -82,6 +82,16 @@ func (c *ClICtrl) PutValue(ctx context.Context, store model.PhotosStore, key []b }) } +func (c *ClICtrl) DeleteValue(ctx context.Context, store model.PhotosStore, key []byte) error { + return c.DB.Update(func(tx *bolt.Tx) error { + kvBucket, err := getAccountStore(ctx, tx, store) + if err != nil { + return err + } + return kvBucket.Delete(key) + }) +} + // GetValue func (c *ClICtrl) GetValue(ctx context.Context, store model.PhotosStore, key []byte) ([]byte, error) { var value []byte From 234131a8860bd04154728975855dd810719c6df5 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 16 Oct 2023 13:29:28 +0530 Subject: [PATCH 089/183] Add support for decrypting file --- internal/crypto/crypto.go | 59 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go index c66409b97..c66a7f8fc 100644 --- a/internal/crypto/crypto.go +++ b/internal/crypto/crypto.go @@ -1,12 +1,14 @@ package crypto import ( + "bufio" "bytes" "cli-go/utils/encoding" "encoding/base64" "fmt" "io" "log" + "os" "github.com/jamesruan/sodium" "golang.org/x/crypto/argon2" @@ -16,6 +18,12 @@ const ( loginSubKeyLen = 32 loginSubKeyId = 1 loginSubKeyContext = "loginctx" + + // xChacha20Poly1305AdditionalBytes indicates the number of additional bytes in each ciphertext block when + // using the secret stream APIs for XChaCha20-Poly1305 encryption/decryption. + //16 bytes for the Poly1305 authentication tag and 1 byte for the chunk type. + xChacha20Poly1305AdditionalBytes = 17 + decryptionBufferSize = 4*1024*1024 + xChacha20Poly1305AdditionalBytes ) // DeriveArgonKey generates a 32-bit cryptographic key using the Argon2id algorithm. @@ -150,3 +158,54 @@ func SealedBoxOpen(cipherText []byte, publicKey, masterSecret []byte) ([]byte, e } return om, nil } + +func DecryptFile(encryptedFilePath string, decryptedFilePath string, key, nonce []byte) error { + inputFile, err := os.Open(encryptedFilePath) + if err != nil { + return err + } + defer inputFile.Close() + + outputFile, err := os.Create(decryptedFilePath) + if err != nil { + return err + } + defer outputFile.Close() + + reader := bufio.NewReader(inputFile) + writer := bufio.NewWriter(outputFile) + + header := sodium.SecretStreamXCPHeader{Bytes: nonce} + decoder, err := sodium.MakeSecretStreamXCPDecoder( + sodium.SecretStreamXCPKey{Bytes: key}, + reader, + header) + if err != nil { + log.Println("Failed to make secret stream decoder", err) + return err + } + + buf := make([]byte, decryptionBufferSize) + for { + n, errErr := decoder.Read(buf) + if errErr != nil && errErr != io.EOF { + log.Println("Failed to read from decoder", err) + return errErr + } + if n == 0 { + break + } + if _, err := writer.Write(buf[:n]); err != nil { + log.Println("Failed to write to output file", err) + return err + } + if errErr == io.EOF { + break + } + } + if err := writer.Flush(); err != nil { + log.Println("Failed to flush writer", err) + return err + } + return nil +} From 78bec602623d78558d77241f28cef0e5d24f802f Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 16 Oct 2023 16:48:23 +0530 Subject: [PATCH 090/183] Download: return path to decrypted file --- pkg/download.go | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/pkg/download.go b/pkg/download.go index 785e0d76e..863f57858 100644 --- a/pkg/download.go +++ b/pkg/download.go @@ -1,28 +1,34 @@ package pkg import ( + "cli-go/internal/crypto" + "cli-go/pkg/model" + "cli-go/utils/encoding" "context" "fmt" + "log" "os" ) -func (c *ClICtrl) initiateDownload(ctx context.Context) error { - files, err := c.getRemoteFiles(ctx) +func (c *ClICtrl) downloadAndDecrypt( + ctx context.Context, + file model.RemoteFile, + deviceKey []byte, +) (*string, error) { + dir, err := os.MkdirTemp("", "ente-cli-download/*") if err != nil { - return err + return nil, err } - dir, err := os.MkdirTemp("", "photos-download") + downloadPath := fmt.Sprintf("%s/%d", dir, file.ID) + log.Printf("Downloading file %d to %s", file.ID, downloadPath) + err = c.Client.DownloadFile(ctx, file.ID, downloadPath) if err != nil { - return err + return nil, err } - fmt.Println("total files to download: in diroector", len(files), dir) - //for _, file := range files { - // //downloadPath := fmt.Sprintf("%s/%d", dir, file.ID) - // //log.Printf("Downloading file %d to %s", file.ID, downloadPath) - // //err = c.Client.DownloadFile(ctx, file.ID, downloadPath) - // //if err != nil { - // // return err - // //} - //} - return nil + decryptedPath := fmt.Sprintf("%s/%d.decrypted", dir, file.ID) + err = crypto.DecryptFile(downloadPath, decryptedPath, file.Key.MustDecrypt(deviceKey), encoding.DecodeBase64(file.FileNonce)) + if err != nil { + return nil, err + } + return &decryptedPath, nil } From b9836f37980c405e2039ed8da87bf65e6e3a6858 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 16 Oct 2023 16:52:45 +0530 Subject: [PATCH 091/183] Export: Write file metadata to disk --- pkg/disk.go | 86 +++++++++++++++ pkg/mapper/photo.go | 5 +- pkg/model/export/metadata.go | 3 + pkg/remote_to_disk_album.go | 42 +------- pkg/remote_to_disk_file.go | 180 +++++++++++++++++++++++++++++++ pkg/remote_to_local_file_sync.go | 63 ----------- pkg/sync.go | 5 - 7 files changed, 273 insertions(+), 111 deletions(-) create mode 100644 pkg/disk.go create mode 100644 pkg/remote_to_disk_file.go delete mode 100644 pkg/remote_to_local_file_sync.go diff --git a/pkg/disk.go b/pkg/disk.go new file mode 100644 index 000000000..422a489eb --- /dev/null +++ b/pkg/disk.go @@ -0,0 +1,86 @@ +package pkg + +import ( + "cli-go/pkg/model" + "cli-go/pkg/model/export" + "context" + "encoding/json" + "fmt" + "os" +) + +const ( + albumMetaFile = "album_meta.json" + albumMetaFolder = ".meta" +) + +type albumDiskInfo struct { + ExportRoot string + AlbumMeta *export.AlbumMetadata + // FileNames contain the name of the files at root level of the album folder + FileNames *map[string]bool + MetaFileNameToDiskFileMap *map[string]*export.DiskFileMetadata + FileIdToDiskFileMap *map[int64]*export.DiskFileMetadata +} + +func (a *albumDiskInfo) IsFilePresent(file model.RemoteFile) bool { + // check if file.ID is present + _, ok := (*a.FileIdToDiskFileMap)[file.ID] + return ok +} +func (a *albumDiskInfo) IsFileNamePresent(fileName string) bool { + _, ok := (*a.FileNames)[fileName] + return ok +} + +func (a *albumDiskInfo) IsMetaFileNamePresent(metaFileName string) bool { + _, ok := (*a.MetaFileNameToDiskFileMap)[metaFileName] + return ok +} + +func (a *albumDiskInfo) GetDiskFile(file model.RemoteFile) *export.DiskFileMetadata { + // check if file.ID is present + diskFile, ok := (*a.FileIdToDiskFileMap)[file.ID] + if !ok { + return nil + } + return diskFile +} + +func writeJSONToFile(filePath string, data interface{}) error { + file, err := os.Create(filePath) + if err != nil { + return err + } + defer file.Close() + + encoder := json.NewEncoder(file) + encoder.SetIndent("", " ") + return encoder.Encode(data) +} + +func readJSONFromFile(filePath string, data interface{}) error { + file, err := os.Open(filePath) + if err != nil { + return err + } + defer file.Close() + + decoder := json.NewDecoder(file) + return decoder.Decode(data) +} + +func exportHome(ctx context.Context) (string, error) { + homeDir, err := os.UserHomeDir() + if err != nil { + return "", err + } + path := fmt.Sprintf("%s/%s", homeDir, "photos") + if _, err = os.Stat(path); os.IsNotExist(err) { + err = os.Mkdir(path, 0755) + if err != nil { + return "", err + } + } + return path, nil +} diff --git a/pkg/mapper/photo.go b/pkg/mapper/photo.go index f35c49b99..d213864cd 100644 --- a/pkg/mapper/photo.go +++ b/pkg/mapper/photo.go @@ -129,8 +129,8 @@ func MapApiFileToPhotoFile(ctx context.Context, album model.RemoteAlbum, file ap return &photoFile, nil } -func MapRemoteFileToDiskMetadata(file model.RemoteFile) export.DiskFileMetadata { - result := export.DiskFileMetadata{ +func MapRemoteFileToDiskMetadata(file model.RemoteFile) *export.DiskFileMetadata { + return &export.DiskFileMetadata{ Title: file.GetTitle(), Description: file.GetCaption(), CreationTime: file.GetCreationTime(), @@ -142,5 +142,4 @@ func MapRemoteFileToDiskMetadata(file model.RemoteFile) export.DiskFileMetadata OwnerID: file.OwnerID, }, } - return result } diff --git a/pkg/model/export/metadata.go b/pkg/model/export/metadata.go index e7ce17bd4..b98454af1 100644 --- a/pkg/model/export/metadata.go +++ b/pkg/model/export/metadata.go @@ -38,6 +38,9 @@ type DiskFileMetadata struct { CreationTime time.Time `json:"creationTime"` ModificationTime time.Time `json:"modificationTime"` Info *Info `json:"info"` + + // exclude this from json serialization + DiskFileName string `json:"-"` } type Info struct { diff --git a/pkg/remote_to_disk_album.go b/pkg/remote_to_disk_album.go index fdd8f28aa..444f0e5fe 100644 --- a/pkg/remote_to_disk_album.go +++ b/pkg/remote_to_disk_album.go @@ -81,7 +81,7 @@ func (c *ClICtrl) createLocalFolderForRemoteAlbums(ctx context.Context) error { } } // Handle meta file - metaFilePath := filepath.Join(path, albumFolderName, ".meta", "album_meta.json") + metaFilePath := filepath.Join(path, albumFolderName, albumMetaFolder, albumMetaFile) metaData := export.AlbumMetadata{ ID: album.ID, OwnerID: album.OwnerID, @@ -112,7 +112,7 @@ func readFolderMetadata(path string) (map[string]*export.AlbumMetadata, map[int6 for _, entry := range entries { if entry.IsDir() { dirName := entry.Name() - metaFilePath := filepath.Join(path, dirName, ".meta", "album_meta.json") + metaFilePath := filepath.Join(path, dirName, albumMetaFolder, albumMetaFile) // Initialize as nil, will remain nil if JSON file is not found or not readable result[dirName] = nil // Read the JSON file if it exists @@ -133,41 +133,3 @@ func readFolderMetadata(path string) (map[string]*export.AlbumMetadata, map[int6 } return result, albumIdToMetadataMap, nil } - -func writeJSONToFile(filePath string, data interface{}) error { - file, err := os.Create(filePath) - if err != nil { - return err - } - defer file.Close() - - encoder := json.NewEncoder(file) - encoder.SetIndent("", " ") - return encoder.Encode(data) -} - -func readJSONFromFile(filePath string, data interface{}) error { - file, err := os.Open(filePath) - if err != nil { - return err - } - defer file.Close() - - decoder := json.NewDecoder(file) - return decoder.Decode(data) -} - -func exportHome(ctx context.Context) (string, error) { - homeDir, err := os.UserHomeDir() - if err != nil { - return "", err - } - path := fmt.Sprintf("%s/%s", homeDir, "photos") - if _, err = os.Stat(path); os.IsNotExist(err) { - err = os.Mkdir(path, 0755) - if err != nil { - return "", err - } - } - return path, nil -} diff --git a/pkg/remote_to_disk_file.go b/pkg/remote_to_disk_file.go new file mode 100644 index 000000000..c7e5e6e7e --- /dev/null +++ b/pkg/remote_to_disk_file.go @@ -0,0 +1,180 @@ +package pkg + +import ( + "cli-go/pkg/mapper" + "cli-go/pkg/model" + "cli-go/pkg/model/export" + "context" + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + "strings" +) + +func (c *ClICtrl) syncFiles(ctx context.Context) error { + log.Printf("Starting sync files") + exportRoot, err := exportHome(ctx) + if err != nil { + return err + } + _, albumIDToMetaMap, err := readFolderMetadata(exportRoot) + if err != nil { + return err + } + entries, err := c.getRemoteAlbumEntries(ctx) + if err != nil { + return err + } + log.Println("total entries", len(entries)) + model.SortAlbumFileEntry(entries) + for i, entry := range entries { + if entry.SyncedLocally { + continue + } + fmt.Println("entry", i, entry.AlbumID, entry.FileID, entry.SyncedLocally) + albumInfo, ok := albumIDToMetaMap[entry.AlbumID] + if !ok { + log.Printf("Album %d not found in local metadata", entry.AlbumID) + continue + } + if albumInfo.IsDeleted { + entry.IsDeleted = true + putErr := c.DeleteValue(ctx, model.RemoteAlbumEntries, []byte(fmt.Sprintf("%d:%d", entry.AlbumID, entry.FileID))) + if putErr != nil { + return putErr + } + continue + } + diskAlbumInfo, err := readFilesMetadata(exportRoot, albumInfo) + if err != nil { + return err + } + fileBytes, err := c.GetValue(ctx, model.RemoteFiles, []byte(fmt.Sprintf("%d", entry.FileID))) + if err != nil { + return err + } + if fileBytes != nil { + var existingEntry *model.RemoteFile + err = json.Unmarshal(fileBytes, &existingEntry) + if err != nil { + return err + } + err = c.downloadEntry(ctx, diskAlbumInfo, *existingEntry, entry) + if err != nil { + return err + } + } else { + log.Fatalf("remoteFile %d not found in remoteFiles", entry.FileID) + } + } + return nil +} + +func (c *ClICtrl) downloadEntry(ctx context.Context, + diskInfo *albumDiskInfo, + file model.RemoteFile, + albumEntry *model.AlbumFileEntry) error { + if !diskInfo.AlbumMeta.IsDeleted && albumEntry.IsDeleted { + albumEntry.IsDeleted = true + diskFile := diskInfo.GetDiskFile(file) + if diskFile != nil { + // remove the file from disk + log.Printf("Removing file %s from disk", diskFile.DiskFileName) + err := os.Remove(filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, ".meta", diskFile.DiskFileName)) + if err != nil { + return err + } + } + putErr := c.DeleteValue(ctx, model.RemoteAlbumEntries, []byte(fmt.Sprintf("%d:%d", albumEntry.AlbumID, albumEntry.FileID))) + if putErr != nil { + return putErr + } + } + if !diskInfo.IsFilePresent(file) { + fileDiskMetadata := mapper.MapRemoteFileToDiskMetadata(file) + potentialDiskFileName := fileDiskMetadata.Title + "." + "json" + count := 1 + for diskInfo.IsMetaFileNamePresent(potentialDiskFileName) { + // separate the file name and extension + potentialDiskFileName = fmt.Sprintf("%s_%d.json", fileDiskMetadata.Title, count) + count++ + if !diskInfo.IsMetaFileNamePresent(potentialDiskFileName) { + break + } + } + err := writeJSONToFile(filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, ".meta", potentialDiskFileName), fileDiskMetadata) + if err != nil { + return err + } + + } + return nil +} + +// readFolderMetadata reads the metadata of the files in the given path +// For disk export, a particular albums files are stored in a folder named after the album. +// Inside the folder, the files are stored at top level and its metadata is stored in a .meta folder +func readFilesMetadata(home string, albumMeta *export.AlbumMetadata) (*albumDiskInfo, error) { + albumMetadataFolder := filepath.Join(home, albumMeta.FolderName, albumMetaFolder) + albumPath := filepath.Join(home, albumMeta.FolderName) + // verify the both the album folder and the .meta folder exist + if _, err := os.Stat(albumMetadataFolder); err != nil { + return nil, err + } + if _, err := os.Stat(albumPath); err != nil { + return nil, err + } + result := make(map[string]*export.DiskFileMetadata) + //fileNameToFileName := make(map[string]*export.DiskFileMetadata) + fileIdToMetadata := make(map[int64]*export.DiskFileMetadata) + claimedFileName := make(map[string]bool) + // Read the top-level directories in the given path + albumFileEntries, err := os.ReadDir(albumPath) + if err != nil { + return nil, err + } + for _, entry := range albumFileEntries { + if !entry.IsDir() { + claimedFileName[entry.Name()] = true + } + } + metaEntries, err := os.ReadDir(albumMetadataFolder) + if err != nil { + return nil, err + } + for _, entry := range metaEntries { + if !entry.IsDir() { + fileName := entry.Name() + if fileName == albumMetaFile { + continue + } + if !strings.HasSuffix(fileName, ".json") { + log.Printf("Skipping file %s as it is not a JSON file", fileName) + continue + } + fileMetadataPath := filepath.Join(albumMetadataFolder, fileName) + // Initialize as nil, will remain nil if JSON file is not found or not readable + result[fileName] = nil + // Read the JSON file if it exists + var metaData export.DiskFileMetadata + metaDataBytes, err := os.ReadFile(fileMetadataPath) + if err != nil { + continue // Skip this entry if reading fails + } + if err := json.Unmarshal(metaDataBytes, &metaData); err == nil { + metaData.DiskFileName = fileName + result[fileName] = &metaData + fileIdToMetadata[metaData.Info.ID] = &metaData + } + } + } + return &albumDiskInfo{ + ExportRoot: home, + AlbumMeta: albumMeta, + FileNames: &claimedFileName, + MetaFileNameToDiskFileMap: &result, + FileIdToDiskFileMap: &fileIdToMetadata, + }, nil +} diff --git a/pkg/remote_to_local_file_sync.go b/pkg/remote_to_local_file_sync.go deleted file mode 100644 index 8bb820eea..000000000 --- a/pkg/remote_to_local_file_sync.go +++ /dev/null @@ -1,63 +0,0 @@ -package pkg - -import ( - "cli-go/pkg/mapper" - "cli-go/pkg/model" - "cli-go/utils/encoding" - "context" - "encoding/json" - "fmt" - "log" -) - -func (c *ClICtrl) syncFiles(ctx context.Context) error { - log.Printf("Starting sync files") - home, err := exportHome(ctx) - if err != nil { - return err - } - _, albumIDToMetaMap, err := readFolderMetadata(home) - if err != nil { - return err - } - entries, err := c.getRemoteAlbumEntries(ctx) - if err != nil { - return err - } - log.Println("total entries", len(entries)) - for _, entry := range entries { - if entry.SyncedLocally { - continue - } - albumInfo, ok := albumIDToMetaMap[entry.AlbumID] - if !ok { - log.Printf("Album %d not found in local metadata", entry.AlbumID) - continue - } - if albumInfo.IsDeleted { - entry.IsDeleted = true - albumEntryJson := encoding.MustMarshalJSON(entry) - putErr := c.PutValue(ctx, model.RemoteAlbumEntries, []byte(fmt.Sprintf("%d:%d", entry.AlbumID, entry.FileID)), albumEntryJson) - if putErr != nil { - return putErr - } - continue - } - fileBytes, err := c.GetValue(ctx, model.RemoteFiles, []byte(fmt.Sprintf("%d", entry.FileID))) - if err != nil { - return err - } - if fileBytes != nil { - var existingEntry *model.RemoteFile - err = json.Unmarshal(fileBytes, &existingEntry) - if err != nil { - return err - } - fileDiskMetadata := mapper.MapRemoteFileToDiskMetadata(*existingEntry) - log.Printf("File %+v\\n ", fileDiskMetadata) - } else { - log.Fatalf("remoteFile %d not found in remoteFiles", entry.FileID) - } - } - return nil -} diff --git a/pkg/sync.go b/pkg/sync.go index f6b12dee4..47012e658 100644 --- a/pkg/sync.go +++ b/pkg/sync.go @@ -63,11 +63,6 @@ func (c *ClICtrl) SyncAccount(account model.Account) error { log.Printf("Error syncing files: %s", err) return err } - downloadErr := c.initiateDownload(ctx) - if downloadErr != nil { - log.Printf("Error downloading files: %s", downloadErr) - return downloadErr - } return nil } From 9eb93f616ae7fb5a1fb05df5773bdeeec7f79bc2 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 16 Oct 2023 16:56:34 +0530 Subject: [PATCH 092/183] Add helper method to track time taken --- pkg/remote_to_disk_file.go | 3 +++ utils/time.go | 11 +++++++++++ 2 files changed, 14 insertions(+) create mode 100644 utils/time.go diff --git a/pkg/remote_to_disk_file.go b/pkg/remote_to_disk_file.go index c7e5e6e7e..a5e77e6fb 100644 --- a/pkg/remote_to_disk_file.go +++ b/pkg/remote_to_disk_file.go @@ -4,6 +4,7 @@ import ( "cli-go/pkg/mapper" "cli-go/pkg/model" "cli-go/pkg/model/export" + "cli-go/utils" "context" "encoding/json" "fmt" @@ -11,6 +12,7 @@ import ( "os" "path/filepath" "strings" + "time" ) func (c *ClICtrl) syncFiles(ctx context.Context) error { @@ -29,6 +31,7 @@ func (c *ClICtrl) syncFiles(ctx context.Context) error { } log.Println("total entries", len(entries)) model.SortAlbumFileEntry(entries) + defer utils.TimeTrack(time.Now(), "process_files") for i, entry := range entries { if entry.SyncedLocally { continue diff --git a/utils/time.go b/utils/time.go new file mode 100644 index 000000000..fcb61f842 --- /dev/null +++ b/utils/time.go @@ -0,0 +1,11 @@ +package utils + +import ( + "log" + "time" +) + +func TimeTrack(start time.Time, name string) { + elapsed := time.Since(start) + log.Printf("%s took %s", name, elapsed) +} From a017944a150dc5298294c80f4bf77b3ce2ea9ae0 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 16 Oct 2023 17:02:16 +0530 Subject: [PATCH 093/183] Fix: Avoid redundant disk read --- pkg/remote_to_disk_file.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pkg/remote_to_disk_file.go b/pkg/remote_to_disk_file.go index a5e77e6fb..f238865fe 100644 --- a/pkg/remote_to_disk_file.go +++ b/pkg/remote_to_disk_file.go @@ -32,16 +32,18 @@ func (c *ClICtrl) syncFiles(ctx context.Context) error { log.Println("total entries", len(entries)) model.SortAlbumFileEntry(entries) defer utils.TimeTrack(time.Now(), "process_files") + var albumDiskInfo *albumDiskInfo for i, entry := range entries { if entry.SyncedLocally { continue } - fmt.Println("entry", i, entry.AlbumID, entry.FileID, entry.SyncedLocally) + albumInfo, ok := albumIDToMetaMap[entry.AlbumID] if !ok { log.Printf("Album %d not found in local metadata", entry.AlbumID) continue } + if albumInfo.IsDeleted { entry.IsDeleted = true putErr := c.DeleteValue(ctx, model.RemoteAlbumEntries, []byte(fmt.Sprintf("%d:%d", entry.AlbumID, entry.FileID))) @@ -50,9 +52,12 @@ func (c *ClICtrl) syncFiles(ctx context.Context) error { } continue } - diskAlbumInfo, err := readFilesMetadata(exportRoot, albumInfo) - if err != nil { - return err + fmt.Println("entry", i, albumInfo.AlbumName, entry.FileID, entry.SyncedLocally) + if albumDiskInfo == nil || albumDiskInfo.AlbumMeta.ID != albumInfo.ID { + albumDiskInfo, err = readFilesMetadata(exportRoot, albumInfo) + if err != nil { + return err + } } fileBytes, err := c.GetValue(ctx, model.RemoteFiles, []byte(fmt.Sprintf("%d", entry.FileID))) if err != nil { @@ -64,7 +69,7 @@ func (c *ClICtrl) syncFiles(ctx context.Context) error { if err != nil { return err } - err = c.downloadEntry(ctx, diskAlbumInfo, *existingEntry, entry) + err = c.downloadEntry(ctx, albumDiskInfo, *existingEntry, entry) if err != nil { return err } From dc97c896330cb68c32b106fda4e28acc1f6502aa Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 16 Oct 2023 17:28:59 +0530 Subject: [PATCH 094/183] Fix: Bug in handling duplicate metadata file names --- pkg/disk.go | 14 ++++++++++++++ pkg/remote_to_disk_file.go | 14 +++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/pkg/disk.go b/pkg/disk.go index 422a489eb..f5de419bc 100644 --- a/pkg/disk.go +++ b/pkg/disk.go @@ -5,6 +5,7 @@ import ( "cli-go/pkg/model/export" "context" "encoding/json" + "errors" "fmt" "os" ) @@ -28,11 +29,24 @@ func (a *albumDiskInfo) IsFilePresent(file model.RemoteFile) bool { _, ok := (*a.FileIdToDiskFileMap)[file.ID] return ok } + func (a *albumDiskInfo) IsFileNamePresent(fileName string) bool { _, ok := (*a.FileNames)[fileName] return ok } +func (a *albumDiskInfo) AddEntry(metadata *export.DiskFileMetadata) error { + if _, ok := (*a.FileIdToDiskFileMap)[metadata.Info.ID]; ok { + return errors.New("fileID already present") + } + if _, ok := (*a.MetaFileNameToDiskFileMap)[metadata.DiskFileName]; ok { + return errors.New("fileName already present") + } + (*a.MetaFileNameToDiskFileMap)[metadata.DiskFileName] = metadata + (*a.FileIdToDiskFileMap)[metadata.Info.ID] = metadata + return nil +} + func (a *albumDiskInfo) IsMetaFileNamePresent(metaFileName string) bool { _, ok := (*a.MetaFileNameToDiskFileMap)[metaFileName] return ok diff --git a/pkg/remote_to_disk_file.go b/pkg/remote_to_disk_file.go index f238865fe..0377d89cb 100644 --- a/pkg/remote_to_disk_file.go +++ b/pkg/remote_to_disk_file.go @@ -102,17 +102,25 @@ func (c *ClICtrl) downloadEntry(ctx context.Context, } if !diskInfo.IsFilePresent(file) { fileDiskMetadata := mapper.MapRemoteFileToDiskMetadata(file) - potentialDiskFileName := fileDiskMetadata.Title + "." + "json" + // Get the extension + extension := filepath.Ext(fileDiskMetadata.Title) + baseFileName := strings.TrimSuffix(filepath.Base(fileDiskMetadata.Title), extension) + potentialDiskFileName := fmt.Sprintf("%s%s.json", baseFileName, extension) count := 1 for diskInfo.IsMetaFileNamePresent(potentialDiskFileName) { // separate the file name and extension - potentialDiskFileName = fmt.Sprintf("%s_%d.json", fileDiskMetadata.Title, count) + potentialDiskFileName = fmt.Sprintf("%s_%d%s.json", baseFileName, count, extension) count++ if !diskInfo.IsMetaFileNamePresent(potentialDiskFileName) { break } } - err := writeJSONToFile(filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, ".meta", potentialDiskFileName), fileDiskMetadata) + fileDiskMetadata.DiskFileName = potentialDiskFileName + err := diskInfo.AddEntry(fileDiskMetadata) + if err != nil { + return err + } + err = writeJSONToFile(filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, ".meta", potentialDiskFileName), fileDiskMetadata) if err != nil { return err } From 06b143fec83c5fe01edc8c5c74ea1c6b370788cf Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 16 Oct 2023 18:22:57 +0530 Subject: [PATCH 095/183] Fix type cast error --- pkg/model/remote.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/model/remote.go b/pkg/model/remote.go index 3fb0be9e4..4787744d6 100644 --- a/pkg/model/remote.go +++ b/pkg/model/remote.go @@ -69,7 +69,7 @@ func (r *RemoteFile) GetFileType() FileType { if !ok { panic("fileType not found in metadata") } - switch value.(int8) { + switch int8(value.(float64)) { case 0: return Image case 1: From 20563eb741ad2828e4a0406f5c5f4eabe5f4a6e0 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 16 Oct 2023 18:24:00 +0530 Subject: [PATCH 096/183] Hook logic to download file --- pkg/disk.go | 4 ++-- pkg/download.go | 6 +++--- pkg/model/export/metadata.go | 2 +- pkg/remote_to_disk_file.go | 18 ++++++++++++++---- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/pkg/disk.go b/pkg/disk.go index f5de419bc..1804ba23a 100644 --- a/pkg/disk.go +++ b/pkg/disk.go @@ -39,10 +39,10 @@ func (a *albumDiskInfo) AddEntry(metadata *export.DiskFileMetadata) error { if _, ok := (*a.FileIdToDiskFileMap)[metadata.Info.ID]; ok { return errors.New("fileID already present") } - if _, ok := (*a.MetaFileNameToDiskFileMap)[metadata.DiskFileName]; ok { + if _, ok := (*a.MetaFileNameToDiskFileMap)[metadata.MetaFileName]; ok { return errors.New("fileName already present") } - (*a.MetaFileNameToDiskFileMap)[metadata.DiskFileName] = metadata + (*a.MetaFileNameToDiskFileMap)[metadata.MetaFileName] = metadata (*a.FileIdToDiskFileMap)[metadata.Info.ID] = metadata return nil } diff --git a/pkg/download.go b/pkg/download.go index 863f57858..71beafb6c 100644 --- a/pkg/download.go +++ b/pkg/download.go @@ -15,7 +15,7 @@ func (c *ClICtrl) downloadAndDecrypt( file model.RemoteFile, deviceKey []byte, ) (*string, error) { - dir, err := os.MkdirTemp("", "ente-cli-download/*") + dir, err := os.MkdirTemp("", "ente-cli-download") if err != nil { return nil, err } @@ -23,12 +23,12 @@ func (c *ClICtrl) downloadAndDecrypt( log.Printf("Downloading file %d to %s", file.ID, downloadPath) err = c.Client.DownloadFile(ctx, file.ID, downloadPath) if err != nil { - return nil, err + return nil, fmt.Errorf("error downloading file %d: %w", file.ID, err) } decryptedPath := fmt.Sprintf("%s/%d.decrypted", dir, file.ID) err = crypto.DecryptFile(downloadPath, decryptedPath, file.Key.MustDecrypt(deviceKey), encoding.DecodeBase64(file.FileNonce)) if err != nil { - return nil, err + return nil, fmt.Errorf("error decrypting file %d: %w", file.ID, err) } return &decryptedPath, nil } diff --git a/pkg/model/export/metadata.go b/pkg/model/export/metadata.go index b98454af1..123f4d935 100644 --- a/pkg/model/export/metadata.go +++ b/pkg/model/export/metadata.go @@ -40,7 +40,7 @@ type DiskFileMetadata struct { Info *Info `json:"info"` // exclude this from json serialization - DiskFileName string `json:"-"` + MetaFileName string `json:"-"` } type Info struct { diff --git a/pkg/remote_to_disk_file.go b/pkg/remote_to_disk_file.go index 0377d89cb..5212abd8c 100644 --- a/pkg/remote_to_disk_file.go +++ b/pkg/remote_to_disk_file.go @@ -89,8 +89,8 @@ func (c *ClICtrl) downloadEntry(ctx context.Context, diskFile := diskInfo.GetDiskFile(file) if diskFile != nil { // remove the file from disk - log.Printf("Removing file %s from disk", diskFile.DiskFileName) - err := os.Remove(filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, ".meta", diskFile.DiskFileName)) + log.Printf("Removing file %s from disk", diskFile.MetaFileName) + err := os.Remove(filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, ".meta", diskFile.MetaFileName)) if err != nil { return err } @@ -101,6 +101,10 @@ func (c *ClICtrl) downloadEntry(ctx context.Context, } } if !diskInfo.IsFilePresent(file) { + decrypt, err := c.downloadAndDecrypt(ctx, file, c.KeyHolder.DeviceKey) + if err != nil { + return err + } fileDiskMetadata := mapper.MapRemoteFileToDiskMetadata(file) // Get the extension extension := filepath.Ext(fileDiskMetadata.Title) @@ -115,8 +119,14 @@ func (c *ClICtrl) downloadEntry(ctx context.Context, break } } - fileDiskMetadata.DiskFileName = potentialDiskFileName - err := diskInfo.AddEntry(fileDiskMetadata) + fileDiskMetadata.MetaFileName = potentialDiskFileName + err = diskInfo.AddEntry(fileDiskMetadata) + if err != nil { + return err + } + fileName := filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, strings.TrimSuffix(filepath.Base(potentialDiskFileName), ".json")) + // move the decrypt file to fileName + err = os.Rename(*decrypt, fileName) if err != nil { return err } From dd4cfc899af767e6676abcf2789b593910c73760 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 16 Oct 2023 19:01:19 +0530 Subject: [PATCH 097/183] Delete file from local when removed from remote --- pkg/disk.go | 17 ++++++++++++- pkg/model/export/metadata.go | 14 +++++++++++ pkg/remote_to_disk_file.go | 47 ++++++++++++++++++++++++------------ 3 files changed, 61 insertions(+), 17 deletions(-) diff --git a/pkg/disk.go b/pkg/disk.go index 1804ba23a..1b2e5c1fb 100644 --- a/pkg/disk.go +++ b/pkg/disk.go @@ -47,12 +47,27 @@ func (a *albumDiskInfo) AddEntry(metadata *export.DiskFileMetadata) error { return nil } +func (a *albumDiskInfo) RemoveEntry(metadata *export.DiskFileMetadata) error { + if _, ok := (*a.FileIdToDiskFileMap)[metadata.Info.ID]; !ok { + return errors.New("fileID not present") + } + if _, ok := (*a.MetaFileNameToDiskFileMap)[metadata.MetaFileName]; !ok { + return errors.New("fileName not present") + } + delete(*a.MetaFileNameToDiskFileMap, metadata.MetaFileName) + delete(*a.FileIdToDiskFileMap, metadata.Info.ID) + for _, filename := range metadata.Info.FileNames { + delete(*a.FileNames, filename) + } + return nil +} + func (a *albumDiskInfo) IsMetaFileNamePresent(metaFileName string) bool { _, ok := (*a.MetaFileNameToDiskFileMap)[metaFileName] return ok } -func (a *albumDiskInfo) GetDiskFile(file model.RemoteFile) *export.DiskFileMetadata { +func (a *albumDiskInfo) GetDiskFileMetadata(file model.RemoteFile) *export.DiskFileMetadata { // check if file.ID is present diskFile, ok := (*a.FileIdToDiskFileMap)[file.ID] if !ok { diff --git a/pkg/model/export/metadata.go b/pkg/model/export/metadata.go index 123f4d935..90c62bda4 100644 --- a/pkg/model/export/metadata.go +++ b/pkg/model/export/metadata.go @@ -43,8 +43,22 @@ type DiskFileMetadata struct { MetaFileName string `json:"-"` } +func (d *DiskFileMetadata) AddFileName(fileName string) { + if d.Info.FileNames == nil { + d.Info.FileNames = make([]string, 0) + } + for _, ownerID := range d.Info.FileNames { + if ownerID == fileName { + return + } + } + d.Info.FileNames = append(d.Info.FileNames, fileName) +} + type Info struct { ID int64 `json:"id"` Hash *string `json:"hash"` OwnerID int64 `json:"ownerID"` + // A file can contain multiple parts (example: live photos or burst photos) + FileNames []string `json:"fileNames"` } diff --git a/pkg/remote_to_disk_file.go b/pkg/remote_to_disk_file.go index 5212abd8c..dec7e153f 100644 --- a/pkg/remote_to_disk_file.go +++ b/pkg/remote_to_disk_file.go @@ -37,7 +37,6 @@ func (c *ClICtrl) syncFiles(ctx context.Context) error { if entry.SyncedLocally { continue } - albumInfo, ok := albumIDToMetaMap[entry.AlbumID] if !ok { log.Printf("Album %d not found in local metadata", entry.AlbumID) @@ -69,9 +68,11 @@ func (c *ClICtrl) syncFiles(ctx context.Context) error { if err != nil { return err } - err = c.downloadEntry(ctx, albumDiskInfo, *existingEntry, entry) - if err != nil { - return err + if existingEntry.GetFileType() != model.LivePhoto && albumDiskInfo.AlbumMeta.ID == 1580559962519759 { + err = c.downloadEntry(ctx, albumDiskInfo, *existingEntry, entry) + if err != nil { + return err + } } } else { log.Fatalf("remoteFile %d not found in remoteFiles", entry.FileID) @@ -86,13 +87,11 @@ func (c *ClICtrl) downloadEntry(ctx context.Context, albumEntry *model.AlbumFileEntry) error { if !diskInfo.AlbumMeta.IsDeleted && albumEntry.IsDeleted { albumEntry.IsDeleted = true - diskFile := diskInfo.GetDiskFile(file) - if diskFile != nil { - // remove the file from disk - log.Printf("Removing file %s from disk", diskFile.MetaFileName) - err := os.Remove(filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, ".meta", diskFile.MetaFileName)) - if err != nil { - return err + diskFileMeta := diskInfo.GetDiskFileMetadata(file) + if diskFileMeta != nil { + removeErr := removeDiskFile(diskFileMeta, diskInfo) + if removeErr != nil { + return removeErr } } putErr := c.DeleteValue(ctx, model.RemoteAlbumEntries, []byte(fmt.Sprintf("%d:%d", albumEntry.AlbumID, albumEntry.FileID))) @@ -124,21 +123,37 @@ func (c *ClICtrl) downloadEntry(ctx context.Context, if err != nil { return err } - fileName := filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, strings.TrimSuffix(filepath.Base(potentialDiskFileName), ".json")) - // move the decrypt file to fileName - err = os.Rename(*decrypt, fileName) + filePath := filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, strings.TrimSuffix(filepath.Base(potentialDiskFileName), ".json")) + // move the decrypt file to filePath + err = os.Rename(*decrypt, filePath) if err != nil { return err } + fileDiskMetadata.AddFileName(filepath.Base(filePath)) err = writeJSONToFile(filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, ".meta", potentialDiskFileName), fileDiskMetadata) if err != nil { return err } - } return nil } +func removeDiskFile(diskFileMeta *export.DiskFileMetadata, diskInfo *albumDiskInfo) error { + // remove the file from disk + log.Printf("Removing file %s from disk", diskFileMeta.MetaFileName) + err := os.Remove(filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, ".meta", diskFileMeta.MetaFileName)) + if err != nil { + return err + } + for _, fileName := range diskFileMeta.Info.FileNames { + err = os.Remove(filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, fileName)) + if err != nil { + return err + } + } + return diskInfo.RemoveEntry(diskFileMeta) +} + // readFolderMetadata reads the metadata of the files in the given path // For disk export, a particular albums files are stored in a folder named after the album. // Inside the folder, the files are stored at top level and its metadata is stored in a .meta folder @@ -190,7 +205,7 @@ func readFilesMetadata(home string, albumMeta *export.AlbumMetadata) (*albumDisk continue // Skip this entry if reading fails } if err := json.Unmarshal(metaDataBytes, &metaData); err == nil { - metaData.DiskFileName = fileName + metaData.MetaFileName = fileName result[fileName] = &metaData fileIdToMetadata[metaData.Info.ID] = &metaData } From 840e475848fa515f16eef11af7bc34f460b31e89 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 16 Oct 2023 19:17:36 +0530 Subject: [PATCH 098/183] Refactor --- pkg/bolt_store.go | 20 ++++++++++++++++++++ pkg/remote_sync.go | 3 +-- pkg/remote_to_disk_file.go | 21 ++++++++++++++------- 3 files changed, 35 insertions(+), 9 deletions(-) create mode 100644 pkg/bolt_store.go diff --git a/pkg/bolt_store.go b/pkg/bolt_store.go new file mode 100644 index 000000000..b13c3e28b --- /dev/null +++ b/pkg/bolt_store.go @@ -0,0 +1,20 @@ +package pkg + +import ( + "cli-go/pkg/model" + "cli-go/utils/encoding" + "context" + "fmt" +) + +func boltAEKey(entry *model.AlbumFileEntry) []byte { + return []byte(fmt.Sprintf("%d:%d", entry.AlbumID, entry.FileID)) +} + +func (c *ClICtrl) DeleteAlbumEntry(ctx context.Context, entry *model.AlbumFileEntry) error { + return c.DeleteValue(ctx, model.RemoteAlbumEntries, boltAEKey(entry)) +} + +func (c *ClICtrl) UpsertAlbumEntry(ctx context.Context, entry *model.AlbumFileEntry) error { + return c.PutValue(ctx, model.RemoteAlbumEntries, boltAEKey(entry), encoding.MustMarshalJSON(entry)) +} diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index 6b3fb54c7..44e7a31cc 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -93,8 +93,7 @@ func (c *ClICtrl) fetchRemoteFiles(ctx context.Context) error { continue } albumEntry := model.AlbumFileEntry{AlbumID: album.ID, FileID: file.ID, IsDeleted: file.IsDeleted, SyncedLocally: false} - albumEntryJson := encoding.MustMarshalJSON(albumEntry) - putErr := c.PutValue(ctx, model.RemoteAlbumEntries, []byte(fmt.Sprintf("%d:%d", album.ID, file.ID)), albumEntryJson) + putErr := c.UpsertAlbumEntry(ctx, &albumEntry) if putErr != nil { return putErr } diff --git a/pkg/remote_to_disk_file.go b/pkg/remote_to_disk_file.go index dec7e153f..de2937412 100644 --- a/pkg/remote_to_disk_file.go +++ b/pkg/remote_to_disk_file.go @@ -44,14 +44,13 @@ func (c *ClICtrl) syncFiles(ctx context.Context) error { } if albumInfo.IsDeleted { - entry.IsDeleted = true - putErr := c.DeleteValue(ctx, model.RemoteAlbumEntries, []byte(fmt.Sprintf("%d:%d", entry.AlbumID, entry.FileID))) + putErr := c.DeleteAlbumEntry(ctx, entry) if putErr != nil { return putErr } continue } - fmt.Println("entry", i, albumInfo.AlbumName, entry.FileID, entry.SyncedLocally) + if albumDiskInfo == nil || albumDiskInfo.AlbumMeta.ID != albumInfo.ID { albumDiskInfo, err = readFilesMetadata(exportRoot, albumInfo) if err != nil { @@ -69,6 +68,7 @@ func (c *ClICtrl) syncFiles(ctx context.Context) error { return err } if existingEntry.GetFileType() != model.LivePhoto && albumDiskInfo.AlbumMeta.ID == 1580559962519759 { + fmt.Println("entry", i, albumInfo.AlbumName, entry.FileID, entry.SyncedLocally) err = c.downloadEntry(ctx, albumDiskInfo, *existingEntry, entry) if err != nil { return err @@ -84,7 +84,8 @@ func (c *ClICtrl) syncFiles(ctx context.Context) error { func (c *ClICtrl) downloadEntry(ctx context.Context, diskInfo *albumDiskInfo, file model.RemoteFile, - albumEntry *model.AlbumFileEntry) error { + albumEntry *model.AlbumFileEntry, +) error { if !diskInfo.AlbumMeta.IsDeleted && albumEntry.IsDeleted { albumEntry.IsDeleted = true diskFileMeta := diskInfo.GetDiskFileMetadata(file) @@ -94,10 +95,11 @@ func (c *ClICtrl) downloadEntry(ctx context.Context, return removeErr } } - putErr := c.DeleteValue(ctx, model.RemoteAlbumEntries, []byte(fmt.Sprintf("%d:%d", albumEntry.AlbumID, albumEntry.FileID))) - if putErr != nil { - return putErr + delErr := c.DeleteAlbumEntry(ctx, albumEntry) + if delErr != nil { + return delErr } + return nil } if !diskInfo.IsFilePresent(file) { decrypt, err := c.downloadAndDecrypt(ctx, file, c.KeyHolder.DeviceKey) @@ -134,6 +136,11 @@ func (c *ClICtrl) downloadEntry(ctx context.Context, if err != nil { return err } + albumEntry.SyncedLocally = true + putErr := c.UpsertAlbumEntry(ctx, albumEntry) + if putErr != nil { + return putErr + } } return nil } From 11f0597a7f8904586fb469622f8a1d0e11de3071 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 16 Oct 2023 19:22:41 +0530 Subject: [PATCH 099/183] Handle file updates by re-downloading data --- pkg/remote_to_disk_file.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/remote_to_disk_file.go b/pkg/remote_to_disk_file.go index de2937412..55791bd58 100644 --- a/pkg/remote_to_disk_file.go +++ b/pkg/remote_to_disk_file.go @@ -101,6 +101,13 @@ func (c *ClICtrl) downloadEntry(ctx context.Context, } return nil } + diskFileMeta := diskInfo.GetDiskFileMetadata(file) + if diskFileMeta != nil { + removeErr := removeDiskFile(diskFileMeta, diskInfo) + if removeErr != nil { + return removeErr + } + } if !diskInfo.IsFilePresent(file) { decrypt, err := c.downloadAndDecrypt(ctx, file, c.KeyHolder.DeviceKey) if err != nil { From cc80b0decb5698f9b590e81ed12cec35d2ea60f6 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 16 Oct 2023 20:52:47 +0530 Subject: [PATCH 100/183] Fix bug in decryption --- internal/crypto/crypto.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go index c66a7f8fc..3f7e80f94 100644 --- a/internal/crypto/crypto.go +++ b/internal/crypto/crypto.go @@ -19,11 +19,7 @@ const ( loginSubKeyId = 1 loginSubKeyContext = "loginctx" - // xChacha20Poly1305AdditionalBytes indicates the number of additional bytes in each ciphertext block when - // using the secret stream APIs for XChaCha20-Poly1305 encryption/decryption. - //16 bytes for the Poly1305 authentication tag and 1 byte for the chunk type. - xChacha20Poly1305AdditionalBytes = 17 - decryptionBufferSize = 4*1024*1024 + xChacha20Poly1305AdditionalBytes + decryptionBufferSize = 4 * 1024 * 1024 ) // DeriveArgonKey generates a 32-bit cryptographic key using the Argon2id algorithm. @@ -189,7 +185,7 @@ func DecryptFile(encryptedFilePath string, decryptedFilePath string, key, nonce for { n, errErr := decoder.Read(buf) if errErr != nil && errErr != io.EOF { - log.Println("Failed to read from decoder", err) + log.Println("Failed to read from decoder", errErr) return errErr } if n == 0 { From c53506359d1c3cfe5935f4524820e2dae2941ea5 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 16 Oct 2023 21:22:48 +0530 Subject: [PATCH 101/183] Add support for download live photos --- pkg/download.go | 57 ++++++++++++++++++++++++++++++++++++++ pkg/model/remote.go | 4 +++ pkg/remote_to_disk_file.go | 41 ++++++++++++++++++++++----- 3 files changed, 95 insertions(+), 7 deletions(-) diff --git a/pkg/download.go b/pkg/download.go index 71beafb6c..0c1d903c3 100644 --- a/pkg/download.go +++ b/pkg/download.go @@ -1,13 +1,17 @@ package pkg import ( + "archive/zip" "cli-go/internal/crypto" "cli-go/pkg/model" "cli-go/utils/encoding" "context" "fmt" + "io" "log" "os" + "path/filepath" + "strings" ) func (c *ClICtrl) downloadAndDecrypt( @@ -32,3 +36,56 @@ func (c *ClICtrl) downloadAndDecrypt( } return &decryptedPath, nil } + +func UnpackLive(src string) (imagePath, videoPath string, retErr error) { + var filenames []string + reader, err := zip.OpenReader(src) + if err != nil { + retErr = err + return + } + defer reader.Close() + + dest := filepath.Dir(src) + + for _, file := range reader.File { + destFilePath := filepath.Join(dest, file.Name) + filenames = append(filenames, destFilePath) + + destDir := filepath.Dir(destFilePath) + if err := os.MkdirAll(destDir, 0755); err != nil { + retErr = err + return + } + + destFile, err := os.Create(destFilePath) + if err != nil { + retErr = err + return + } + defer destFile.Close() + + srcFile, err := file.Open() + if err != nil { + retErr = err + return + } + defer srcFile.Close() + + _, err = io.Copy(destFile, srcFile) + if err != nil { + retErr = err + return + } + } + for _, filepath := range filenames { + if strings.Contains(strings.ToLower(filepath), "image") { + imagePath = filepath + } else if strings.Contains(strings.ToLower(filepath), "video") { + videoPath = filepath + } else { + retErr = fmt.Errorf("unexpcted file in zip %s", filepath) + } + } + return +} diff --git a/pkg/model/remote.go b/pkg/model/remote.go index 4787744d6..32fb122cd 100644 --- a/pkg/model/remote.go +++ b/pkg/model/remote.go @@ -80,6 +80,10 @@ func (r *RemoteFile) GetFileType() FileType { panic(fmt.Sprintf("invalid fileType %d", value.(int8))) } +func (r *RemoteFile) IsLivePhoto() bool { + return r.GetFileType() == LivePhoto +} + func (r *RemoteFile) GetFileHash() *string { value, ok := r.Metadata["hash"] if !ok { diff --git a/pkg/remote_to_disk_file.go b/pkg/remote_to_disk_file.go index 55791bd58..fd19e0df3 100644 --- a/pkg/remote_to_disk_file.go +++ b/pkg/remote_to_disk_file.go @@ -121,7 +121,8 @@ func (c *ClICtrl) downloadEntry(ctx context.Context, count := 1 for diskInfo.IsMetaFileNamePresent(potentialDiskFileName) { // separate the file name and extension - potentialDiskFileName = fmt.Sprintf("%s_%d%s.json", baseFileName, count, extension) + baseFileName = fmt.Sprintf("%s_%d", baseFileName, count) + potentialDiskFileName = fmt.Sprintf("%s%s.json", baseFileName, extension) count++ if !diskInfo.IsMetaFileNamePresent(potentialDiskFileName) { break @@ -132,13 +133,39 @@ func (c *ClICtrl) downloadEntry(ctx context.Context, if err != nil { return err } - filePath := filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, strings.TrimSuffix(filepath.Base(potentialDiskFileName), ".json")) - // move the decrypt file to filePath - err = os.Rename(*decrypt, filePath) - if err != nil { - return err + if file.IsLivePhoto() { + imagePath, videoPath, err := UnpackLive(*decrypt) + if err != nil { + return err + } + imageExtn := filepath.Ext(imagePath) + videoExtn := filepath.Ext(videoPath) + imageFileName := baseFileName + imageExtn + videoFileName := baseFileName + videoExtn + imageFilePath := filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, imageFileName) + videoFilePath := filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, videoFileName) + // move the decrypt file to filePath + err = os.Rename(imagePath, imageFilePath) + if err != nil { + return err + } + err = os.Rename(videoPath, videoFilePath) + if err != nil { + return err + } + fileDiskMetadata.AddFileName(imageFileName) + fileDiskMetadata.AddFileName(videoFileName) + } else { + fileName := baseFileName + extension + filePath := filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, fileName) + // move the decrypt file to filePath + err = os.Rename(*decrypt, filePath) + if err != nil { + return err + } + fileDiskMetadata.AddFileName(fileName) } - fileDiskMetadata.AddFileName(filepath.Base(filePath)) + err = writeJSONToFile(filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, ".meta", potentialDiskFileName), fileDiskMetadata) if err != nil { return err From 775a3b472186f2742fa95dea1e3de119f8e863f8 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 16 Oct 2023 21:57:31 +0530 Subject: [PATCH 102/183] Use same temp folder for download --- pkg/cli.go | 13 ++++++++++--- pkg/download.go | 7 ++----- pkg/remote_to_disk_file.go | 10 ++++------ 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/pkg/cli.go b/pkg/cli.go index ee80660c7..5bcc8edc3 100644 --- a/pkg/cli.go +++ b/pkg/cli.go @@ -5,15 +5,22 @@ import ( "cli-go/pkg/secrets" "fmt" bolt "go.etcd.io/bbolt" + "os" ) type ClICtrl struct { - Client *api.Client - DB *bolt.DB - KeyHolder *secrets.KeyHolder + Client *api.Client + DB *bolt.DB + KeyHolder *secrets.KeyHolder + tempFolder string } func (c *ClICtrl) Init() error { + dir, err := os.MkdirTemp("", "ente-cli-download") + if err != nil { + return err + } + c.tempFolder = dir return c.DB.Update(func(tx *bolt.Tx) error { _, err := tx.CreateBucketIfNotExists([]byte(AccBucket)) if err != nil { diff --git a/pkg/download.go b/pkg/download.go index 0c1d903c3..2e456b92b 100644 --- a/pkg/download.go +++ b/pkg/download.go @@ -19,13 +19,10 @@ func (c *ClICtrl) downloadAndDecrypt( file model.RemoteFile, deviceKey []byte, ) (*string, error) { - dir, err := os.MkdirTemp("", "ente-cli-download") - if err != nil { - return nil, err - } + dir := c.tempFolder downloadPath := fmt.Sprintf("%s/%d", dir, file.ID) log.Printf("Downloading file %d to %s", file.ID, downloadPath) - err = c.Client.DownloadFile(ctx, file.ID, downloadPath) + err := c.Client.DownloadFile(ctx, file.ID, downloadPath) if err != nil { return nil, fmt.Errorf("error downloading file %d: %w", file.ID, err) } diff --git a/pkg/remote_to_disk_file.go b/pkg/remote_to_disk_file.go index fd19e0df3..cdec43c80 100644 --- a/pkg/remote_to_disk_file.go +++ b/pkg/remote_to_disk_file.go @@ -67,12 +67,10 @@ func (c *ClICtrl) syncFiles(ctx context.Context) error { if err != nil { return err } - if existingEntry.GetFileType() != model.LivePhoto && albumDiskInfo.AlbumMeta.ID == 1580559962519759 { - fmt.Println("entry", i, albumInfo.AlbumName, entry.FileID, entry.SyncedLocally) - err = c.downloadEntry(ctx, albumDiskInfo, *existingEntry, entry) - if err != nil { - return err - } + log.Printf("[%d/%d] Sync %s for album %s", i, len(entries), existingEntry.GetTitle(), albumInfo.AlbumName) + err = c.downloadEntry(ctx, albumDiskInfo, *existingEntry, entry) + if err != nil { + return err } } else { log.Fatalf("remoteFile %d not found in remoteFiles", entry.FileID) From fd1a7b25489cb82847e38b559bdf57c4c9bab7e4 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 16 Oct 2023 23:04:12 +0530 Subject: [PATCH 103/183] Add support for configuring export directory --- cmd/account.go | 43 ++++++++++++++++++++++++++++++++++++++++-- pkg/account.go | 45 +++++++++++++++++++++++++++++++++++++++++--- pkg/disk.go | 31 ++++++++++++++++++++++++++++++ pkg/model/account.go | 7 +++++++ 4 files changed, 121 insertions(+), 5 deletions(-) diff --git a/cmd/account.go b/cmd/account.go index 0a1e30f87..db4bca3a7 100644 --- a/cmd/account.go +++ b/cmd/account.go @@ -1,8 +1,10 @@ package cmd import ( + "cli-go/internal/api" + "cli-go/pkg/model" "context" - + "fmt" "github.com/spf13/cobra" ) @@ -32,9 +34,46 @@ var addAccCmd = &cobra.Command{ }, } +// Subcommand for 'account update' +var updateAccCmd = &cobra.Command{ + Use: "update", + Short: "Update an existing account's export directory", + Run: func(cmd *cobra.Command, args []string) { + recoverWithLog() + exportDir, _ := cmd.Flags().GetString("dir") + app, _ := cmd.Flags().GetString("app") + email, _ := cmd.Flags().GetString("email") + if email == "" { + fmt.Printf("email must be specified") + } + + validApps := map[string]bool{ + "photos": true, + "locker": true, + "auth": true, + } + + if !validApps[app] { + fmt.Printf("invalid app. Accepted values are 'photos', 'locker', 'auth'") + + } + err := ctrl.UpdateAccount(context.Background(), model.UpdateAccountParams{ + Email: email, + App: api.StringToApp(app), + ExportDir: &exportDir, + }) + if err != nil { + fmt.Printf("Error updating account: %v\n", err) + } + }, +} + func init() { // Add 'config' subcommands to the root command rootCmd.AddCommand(accountCmd) // Add 'config' subcommands to the 'config' command - accountCmd.AddCommand(listAccCmd, addAccCmd) + updateAccCmd.Flags().String("dir", "", "Update export directory") + updateAccCmd.Flags().String("email", "", "email address of the account to update") + updateAccCmd.Flags().String("app", "photos", "Specify the app, default is 'photos'") + accountCmd.AddCommand(listAccCmd, addAccCmd, updateAccCmd) } diff --git a/pkg/account.go b/pkg/account.go index f11bbd6a2..6ac1e6018 100644 --- a/pkg/account.go +++ b/pkg/account.go @@ -125,10 +125,49 @@ func (c *ClICtrl) ListAccounts(cxt context.Context) error { fmt.Printf("Configured accounts: %d\n", len(accounts)) for _, acc := range accounts { fmt.Println("====================================") - fmt.Println("Email: ", acc.Email) - fmt.Println("ID: ", acc.UserID) - fmt.Println("App: ", acc.App) + fmt.Println("Email: ", acc.Email) + fmt.Println("ID: ", acc.UserID) + fmt.Println("App: ", acc.App) + fmt.Println("ExportDir:", acc.ExportDir) fmt.Println("====================================") } return nil } + +func (c *ClICtrl) UpdateAccount(ctx context.Context, params model.UpdateAccountParams) error { + accounts, err := c.GetAccounts(ctx) + if err != nil { + return err + } + var acc *model.Account + for _, a := range accounts { + if a.Email == params.Email && a.App == params.App { + acc = &a + break + } + } + if acc == nil { + return fmt.Errorf("account not found") + } + if params.ExportDir != nil && *params.ExportDir != "" { + _, err := validateExportDirectory(*params.ExportDir) + if err != nil { + return err + } + acc.ExportDir = *params.ExportDir + } + err = c.DB.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucketIfNotExists([]byte(AccBucket)) + if err != nil { + return err + } + accInfoBytes, err := json.Marshal(acc) + if err != nil { + return err + } + accountKey := acc.AccountKey() + return b.Put([]byte(accountKey), accInfoBytes) + }) + return err + +} diff --git a/pkg/disk.go b/pkg/disk.go index 1b2e5c1fb..543cbdeb9 100644 --- a/pkg/disk.go +++ b/pkg/disk.go @@ -113,3 +113,34 @@ func exportHome(ctx context.Context) (string, error) { } return path, nil } + +func validateExportDirectory(dir string) (bool, error) { + // Check if the path exists + fileInfo, err := os.Stat(dir) + if err != nil { + if os.IsNotExist(err) { + return false, fmt.Errorf("path does not exist: %s", dir) + } + return false, err + } + + // Check if the path is a directory + if !fileInfo.IsDir() { + return false, fmt.Errorf("path is not a directory") + } + + // Check for write permission + file, err := os.OpenFile(dir, os.O_WRONLY, 0666) + if err != nil { + if os.IsPermission(err) { + return false, fmt.Errorf("write permission denied") + } + return false, err + } + err = file.Close() + if err != nil { + return false, err + } + + return true, nil +} diff --git a/pkg/model/account.go b/pkg/model/account.go index f721835cb..9f171c4f6 100644 --- a/pkg/model/account.go +++ b/pkg/model/account.go @@ -14,6 +14,13 @@ type Account struct { // PublicKey corresponding to the secret key PublicKey string `json:"publicKey" binding:"required"` Token EncString `json:"token" binding:"required"` + ExportDir string `json:"exportDir"` +} + +type UpdateAccountParams struct { + Email string + App api.App + ExportDir *string } func (a *Account) AccountKey() string { From dca3c4e2c4762ccd128f282bec891cfde2615dc5 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 16 Oct 2023 23:18:00 +0530 Subject: [PATCH 104/183] Fix exportDir validation --- pkg/disk.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/disk.go b/pkg/disk.go index 543cbdeb9..418aa703e 100644 --- a/pkg/disk.go +++ b/pkg/disk.go @@ -130,14 +130,14 @@ func validateExportDirectory(dir string) (bool, error) { } // Check for write permission - file, err := os.OpenFile(dir, os.O_WRONLY, 0666) + // Check for write permission by creating a temp file + tempFile, err := os.CreateTemp(dir, "write_test_") if err != nil { - if os.IsPermission(err) { - return false, fmt.Errorf("write permission denied") - } - return false, err + return false, fmt.Errorf("write permission denied: %v", err) } - err = file.Close() + + // Delete temp file + defer os.Remove(tempFile.Name()) if err != nil { return false, err } From 6285220c1b3bb694d1e17e4f847b4709985102e9 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 16 Oct 2023 23:19:35 +0530 Subject: [PATCH 105/183] Valid and use exportDir during sync --- pkg/disk.go | 16 ---------------- pkg/remote_to_disk_album.go | 9 +++------ pkg/remote_to_disk_file.go | 7 ++----- pkg/sync.go | 13 +++++++++++-- 4 files changed, 16 insertions(+), 29 deletions(-) diff --git a/pkg/disk.go b/pkg/disk.go index 418aa703e..67568dc27 100644 --- a/pkg/disk.go +++ b/pkg/disk.go @@ -3,7 +3,6 @@ package pkg import ( "cli-go/pkg/model" "cli-go/pkg/model/export" - "context" "encoding/json" "errors" "fmt" @@ -99,21 +98,6 @@ func readJSONFromFile(filePath string, data interface{}) error { return decoder.Decode(data) } -func exportHome(ctx context.Context) (string, error) { - homeDir, err := os.UserHomeDir() - if err != nil { - return "", err - } - path := fmt.Sprintf("%s/%s", homeDir, "photos") - if _, err = os.Stat(path); os.IsNotExist(err) { - err = os.Mkdir(path, 0755) - if err != nil { - return "", err - } - } - return path, nil -} - func validateExportDirectory(dir string) (bool, error) { // Check if the path exists fileInfo, err := os.Stat(dir) diff --git a/pkg/remote_to_disk_album.go b/pkg/remote_to_disk_album.go index 444f0e5fe..cf8c57434 100644 --- a/pkg/remote_to_disk_album.go +++ b/pkg/remote_to_disk_album.go @@ -1,6 +1,7 @@ package pkg import ( + "cli-go/pkg/model" "cli-go/pkg/model/export" "context" "encoding/json" @@ -12,12 +13,8 @@ import ( "path/filepath" ) -func (c *ClICtrl) createLocalFolderForRemoteAlbums(ctx context.Context) error { - path, pathErr := exportHome(ctx) - if pathErr != nil { - return pathErr - } - +func (c *ClICtrl) createLocalFolderForRemoteAlbums(ctx context.Context, account model.Account) error { + path := account.ExportDir albums, err := c.getRemoteAlbums(ctx) if err != nil { return err diff --git a/pkg/remote_to_disk_file.go b/pkg/remote_to_disk_file.go index cdec43c80..16ade9fd9 100644 --- a/pkg/remote_to_disk_file.go +++ b/pkg/remote_to_disk_file.go @@ -15,12 +15,9 @@ import ( "time" ) -func (c *ClICtrl) syncFiles(ctx context.Context) error { +func (c *ClICtrl) syncFiles(ctx context.Context, account model.Account) error { log.Printf("Starting sync files") - exportRoot, err := exportHome(ctx) - if err != nil { - return err - } + exportRoot := account.ExportDir _, albumIDToMetaMap, err := readFolderMetadata(exportRoot) if err != nil { return err diff --git a/pkg/sync.go b/pkg/sync.go index 47012e658..f87826bb0 100644 --- a/pkg/sync.go +++ b/pkg/sync.go @@ -21,6 +21,15 @@ func (c *ClICtrl) StartSync() error { } for _, account := range accounts { log.SetPrefix(fmt.Sprintf("[%s-%s] ", account.App, account.Email)) + if account.ExportDir == "" { + log.Printf("Skip account %s: no export directory configured", account.Email) + continue + } + _, err = validateExportDirectory(account.ExportDir) + if err != nil { + log.Printf("Skip export, error: %v while validing exportDir %s\n", err, account.ExportDir) + continue + } log.Println("start sync") err = c.SyncAccount(account) if err != nil { @@ -53,12 +62,12 @@ func (c *ClICtrl) SyncAccount(account model.Account) error { if err != nil { log.Printf("Error fetching files: %s", err) } - err = c.createLocalFolderForRemoteAlbums(ctx) + err = c.createLocalFolderForRemoteAlbums(ctx, account) if err != nil { log.Printf("Error creating local folders: %s", err) return err } - err = c.syncFiles(ctx) + err = c.syncFiles(ctx, account) if err != nil { log.Printf("Error syncing files: %s", err) return err From ae1b1917658f221b64bba25ff88ae32e258e08ef Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 16 Oct 2023 23:26:31 +0530 Subject: [PATCH 106/183] Improve logging --- pkg/download.go | 3 ++- utils/time.go | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/pkg/download.go b/pkg/download.go index 2e456b92b..05b991dcc 100644 --- a/pkg/download.go +++ b/pkg/download.go @@ -4,6 +4,7 @@ import ( "archive/zip" "cli-go/internal/crypto" "cli-go/pkg/model" + "cli-go/utils" "cli-go/utils/encoding" "context" "fmt" @@ -21,7 +22,7 @@ func (c *ClICtrl) downloadAndDecrypt( ) (*string, error) { dir := c.tempFolder downloadPath := fmt.Sprintf("%s/%d", dir, file.ID) - log.Printf("Downloading file %d to %s", file.ID, downloadPath) + log.Printf("Downloading %s (%s)", file.GetTitle(), utils.ByteCountDecimal(file.Info.FileSize)) err := c.Client.DownloadFile(ctx, file.ID, downloadPath) if err != nil { return nil, fmt.Errorf("error downloading file %d: %w", file.ID, err) diff --git a/utils/time.go b/utils/time.go index fcb61f842..a86a9fc9f 100644 --- a/utils/time.go +++ b/utils/time.go @@ -1,6 +1,7 @@ package utils import ( + "fmt" "log" "time" ) @@ -9,3 +10,16 @@ func TimeTrack(start time.Time, name string) { elapsed := time.Since(start) log.Printf("%s took %s", name, elapsed) } + +func ByteCountDecimal(b int64) string { + const unit = 1000 + if b < unit { + return fmt.Sprintf("%d B", b) + } + div, exp := int64(unit), 0 + for n := b / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMGTPE"[exp]) +} From 5a97e36ccb2ccaa6f74bb9a98ae5c1ccb8dbb669 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 16 Oct 2023 23:48:00 +0530 Subject: [PATCH 107/183] Promot for export dir during add --- internal/promt.go | 67 ++++++++++++++++++++++++++++++++++++++++++++++ pkg/account.go | 13 ++++++--- pkg/disk.go | 32 ---------------------- pkg/remote_sync.go | 1 - pkg/sync.go | 3 ++- 5 files changed, 79 insertions(+), 37 deletions(-) diff --git a/internal/promt.go b/internal/promt.go index 5a9145333..e616934b3 100644 --- a/internal/promt.go +++ b/internal/promt.go @@ -75,3 +75,70 @@ func GetCode(promptText string, length int) (string, error) { return ott, nil } } + +func GetExportDir() string { + for { + exportDir, err := GetUserInput("Enter export directory") + if err != nil { + return "" + } + if exportDir == "" { + fmt.Printf("invalid export directory: %s\n", err) + continue + } + exportDir, err = ResolvePath(exportDir) + if err != nil { + fmt.Printf("invalid export directory: %s\n", err) + continue + } + _, err = ValidateDirForWrite(exportDir) + if err != nil { + fmt.Printf("invalid export directory: %s\n", err) + continue + } + + return exportDir + } +} + +func ValidateDirForWrite(dir string) (bool, error) { + // Check if the path exists + fileInfo, err := os.Stat(dir) + if err != nil { + if os.IsNotExist(err) { + return false, fmt.Errorf("path does not exist: %s", dir) + } + return false, err + } + + // Check if the path is a directory + if !fileInfo.IsDir() { + return false, fmt.Errorf("path is not a directory") + } + + // Check for write permission + // Check for write permission by creating a temp file + tempFile, err := os.CreateTemp(dir, "write_test_") + if err != nil { + return false, fmt.Errorf("write permission denied: %v", err) + } + + // Delete temp file + defer os.Remove(tempFile.Name()) + if err != nil { + return false, err + } + + return true, nil +} + +func ResolvePath(path string) (string, error) { + if path[:2] != "~/" { + return path, nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + return home + path[1:], nil +} diff --git a/pkg/account.go b/pkg/account.go index 6ac1e6018..7d5524f97 100644 --- a/pkg/account.go +++ b/pkg/account.go @@ -24,6 +24,11 @@ func (c *ClICtrl) AddAccount(cxt context.Context) { }() app := internal.GetAppType() cxt = context.WithValue(cxt, "app", string(app)) + dir := internal.GetExportDir() + if dir == "" { + flowErr = fmt.Errorf("export directory not set") + return + } email, flowErr := internal.GetUserInput("Enter email address") if flowErr != nil { return @@ -61,7 +66,8 @@ func (c *ClICtrl) AddAccount(cxt context.Context) { flowErr = decErr return } - err := c.storeAccount(cxt, email, authResponse.ID, app, secretInfo) + + err := c.storeAccount(cxt, email, authResponse.ID, app, secretInfo, dir) if err != nil { flowErr = err return @@ -70,7 +76,7 @@ func (c *ClICtrl) AddAccount(cxt context.Context) { } } -func (c *ClICtrl) storeAccount(_ context.Context, email string, userID int64, app api.App, secretInfo *model.AccSecretInfo) error { +func (c *ClICtrl) storeAccount(_ context.Context, email string, userID int64, app api.App, secretInfo *model.AccSecretInfo, exportDir string) error { // get password err := c.DB.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucketIfNotExists([]byte(AccBucket)) @@ -85,6 +91,7 @@ func (c *ClICtrl) storeAccount(_ context.Context, email string, userID int64, ap Token: *model.MakeEncString(secretInfo.Token, c.KeyHolder.DeviceKey), App: app, PublicKey: encoding.EncodeBase64(secretInfo.PublicKey), + ExportDir: exportDir, } accInfoBytes, err := json.Marshal(accInfo) if err != nil { @@ -150,7 +157,7 @@ func (c *ClICtrl) UpdateAccount(ctx context.Context, params model.UpdateAccountP return fmt.Errorf("account not found") } if params.ExportDir != nil && *params.ExportDir != "" { - _, err := validateExportDirectory(*params.ExportDir) + _, err := internal.ValidateDirForWrite(*params.ExportDir) if err != nil { return err } diff --git a/pkg/disk.go b/pkg/disk.go index 67568dc27..adb9345fc 100644 --- a/pkg/disk.go +++ b/pkg/disk.go @@ -5,7 +5,6 @@ import ( "cli-go/pkg/model/export" "encoding/json" "errors" - "fmt" "os" ) @@ -97,34 +96,3 @@ func readJSONFromFile(filePath string, data interface{}) error { decoder := json.NewDecoder(file) return decoder.Decode(data) } - -func validateExportDirectory(dir string) (bool, error) { - // Check if the path exists - fileInfo, err := os.Stat(dir) - if err != nil { - if os.IsNotExist(err) { - return false, fmt.Errorf("path does not exist: %s", dir) - } - return false, err - } - - // Check if the path is a directory - if !fileInfo.IsDir() { - return false, fmt.Errorf("path is not a directory") - } - - // Check for write permission - // Check for write permission by creating a temp file - tempFile, err := os.CreateTemp(dir, "write_test_") - if err != nil { - return false, fmt.Errorf("write permission denied: %v", err) - } - - // Delete temp file - defer os.Remove(tempFile.Name()) - if err != nil { - return false, err - } - - return true, nil -} diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index 44e7a31cc..0b1886473 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -55,7 +55,6 @@ func (c *ClICtrl) fetchRemoteFiles(ctx context.Context) error { } for _, album := range albums { if album.IsDeleted { - log.Printf("Skipping album %s as it is deleted", album.AlbumName) continue } diff --git a/pkg/sync.go b/pkg/sync.go index f87826bb0..504ffbd3d 100644 --- a/pkg/sync.go +++ b/pkg/sync.go @@ -1,6 +1,7 @@ package pkg import ( + "cli-go/internal" "cli-go/pkg/model" "context" "encoding/base64" @@ -25,7 +26,7 @@ func (c *ClICtrl) StartSync() error { log.Printf("Skip account %s: no export directory configured", account.Email) continue } - _, err = validateExportDirectory(account.ExportDir) + _, err = internal.ValidateDirForWrite(account.ExportDir) if err != nil { log.Printf("Skip export, error: %v while validing exportDir %s\n", err, account.ExportDir) continue From 341f032554ebe33c9b598e0bdc344cbb85699213 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 18 Oct 2023 12:17:30 +0530 Subject: [PATCH 108/183] Clean up Dockerfile --- Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 24ab96737..522350be9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,9 +16,6 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ FROM alpine:3.17 RUN apk add libsodium-dev COPY --from=builder /etc/ente/ente-cli . -#COPY configurations configurations -#COPY migrations migrations -#COPY mail-templates mail-templates ARG GIT_COMMIT ENV GIT_COMMIT=$GIT_COMMIT From 28ddb06e2b2644f18e6f3c0818d64a8344b9205c Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 18 Oct 2023 12:17:46 +0530 Subject: [PATCH 109/183] Add export command --- cmd/export.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 cmd/export.go diff --git a/cmd/export.go b/cmd/export.go new file mode 100644 index 000000000..fb850c281 --- /dev/null +++ b/cmd/export.go @@ -0,0 +1,19 @@ +package cmd + +import ( + "github.com/spf13/cobra" +) + +// versionCmd represents the version command +var exportCmd = &cobra.Command{ + Use: "export", + Short: "Starts the export process", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + ctrl.StartSync() + }, +} + +func init() { + rootCmd.AddCommand(exportCmd) +} From de19f450a8f1d4ff517f9ffca5744d1c8baf2b7d Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 18 Oct 2023 12:22:28 +0530 Subject: [PATCH 110/183] Continue export during decryption failure --- cmd/export.go | 2 +- main.go | 1 - pkg/download.go | 3 ++- pkg/model/errors.go | 5 +++++ pkg/remote_to_disk_file.go | 7 ++++++- pkg/sync.go | 4 ++-- 6 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 pkg/model/errors.go diff --git a/cmd/export.go b/cmd/export.go index fb850c281..57548c4fe 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -10,7 +10,7 @@ var exportCmd = &cobra.Command{ Short: "Starts the export process", Long: ``, Run: func(cmd *cobra.Command, args []string) { - ctrl.StartSync() + ctrl.Export() }, } diff --git a/main.go b/main.go index 51a00d0cd..168fb921d 100644 --- a/main.go +++ b/main.go @@ -30,5 +30,4 @@ func main() { } }() cmd.Execute(&ctrl) - //ctrl.StartSync() } diff --git a/pkg/download.go b/pkg/download.go index 05b991dcc..70a98a018 100644 --- a/pkg/download.go +++ b/pkg/download.go @@ -30,7 +30,8 @@ func (c *ClICtrl) downloadAndDecrypt( decryptedPath := fmt.Sprintf("%s/%d.decrypted", dir, file.ID) err = crypto.DecryptFile(downloadPath, decryptedPath, file.Key.MustDecrypt(deviceKey), encoding.DecodeBase64(file.FileNonce)) if err != nil { - return nil, fmt.Errorf("error decrypting file %d: %w", file.ID, err) + log.Printf("Error decrypting file %d: %s", file.ID, err) + return nil, model.ErrDecryption } return &decryptedPath, nil } diff --git a/pkg/model/errors.go b/pkg/model/errors.go new file mode 100644 index 000000000..7f041ef6e --- /dev/null +++ b/pkg/model/errors.go @@ -0,0 +1,5 @@ +package model + +import "errors" + +var ErrDecryption = errors.New("error while decrypting the file") diff --git a/pkg/remote_to_disk_file.go b/pkg/remote_to_disk_file.go index 16ade9fd9..3dbe27bf7 100644 --- a/pkg/remote_to_disk_file.go +++ b/pkg/remote_to_disk_file.go @@ -7,6 +7,7 @@ import ( "cli-go/utils" "context" "encoding/json" + "errors" "fmt" "log" "os" @@ -67,7 +68,11 @@ func (c *ClICtrl) syncFiles(ctx context.Context, account model.Account) error { log.Printf("[%d/%d] Sync %s for album %s", i, len(entries), existingEntry.GetTitle(), albumInfo.AlbumName) err = c.downloadEntry(ctx, albumDiskInfo, *existingEntry, entry) if err != nil { - return err + if errors.Is(err, model.ErrDecryption) { + continue + } else { + return err + } } } else { log.Fatalf("remoteFile %d not found in remoteFiles", entry.FileID) diff --git a/pkg/sync.go b/pkg/sync.go index 504ffbd3d..f0b822d14 100644 --- a/pkg/sync.go +++ b/pkg/sync.go @@ -11,13 +11,13 @@ import ( bolt "go.etcd.io/bbolt" ) -func (c *ClICtrl) StartSync() error { +func (c *ClICtrl) Export() error { accounts, err := c.GetAccounts(context.Background()) if err != nil { return err } if len(accounts) == 0 { - fmt.Printf("No accounts to sync\n") + fmt.Printf("No accounts to sync\n Add account using `account add` cmd\n") return nil } for _, account := range accounts { From 1cb5414e797e2a6d4205bdd9599b838136ae92ec Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 18 Oct 2023 12:22:38 +0530 Subject: [PATCH 111/183] Remove config cmd --- cmd/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/config.go b/cmd/config.go index 896edec53..8a34b4054 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -45,7 +45,7 @@ func init() { viper.SetDefault("host", "https://api.ente.io") // Add 'config' subcommands to the root command - rootCmd.AddCommand(configCmd) + //rootCmd.AddCommand(configCmd) // Add flags to the 'config store' and 'config update' subcommands updateCmd.Flags().StringVarP(&host, "host", "H", viper.GetString("host"), "Update the 'host' configuration") From fd7aac9b443fa5da743b92600b95475cc3b635d8 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 18 Oct 2023 12:41:20 +0530 Subject: [PATCH 112/183] Update account cmd and documentation --- README.md | 11 ++++++++++- cmd/account.go | 9 +++++++-- pkg/account.go | 2 +- pkg/sync.go | 5 +++++ release.sh | 4 ++++ 5 files changed, 27 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 287a14b85..641213f1f 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,17 @@ ```shell ente-cli account list ``` + +* Change export directory + ```shell + ente-cli account update --email yourEmail@example.com --dir ~/photos + ``` - +## Export +* Start export + ```shell + ente-cli export + ``` ## Testing diff --git a/cmd/account.go b/cmd/account.go index db4bca3a7..5704a4de7 100644 --- a/cmd/account.go +++ b/cmd/account.go @@ -44,7 +44,12 @@ var updateAccCmd = &cobra.Command{ app, _ := cmd.Flags().GetString("app") email, _ := cmd.Flags().GetString("email") if email == "" { - fmt.Printf("email must be specified") + fmt.Println("email must be specified") + return + } + if exportDir == "" { + fmt.Println("dir param must be specified") + return } validApps := map[string]bool{ @@ -72,7 +77,7 @@ func init() { // Add 'config' subcommands to the root command rootCmd.AddCommand(accountCmd) // Add 'config' subcommands to the 'config' command - updateAccCmd.Flags().String("dir", "", "Update export directory") + updateAccCmd.Flags().String("dir", "", "update export directory") updateAccCmd.Flags().String("email", "", "email address of the account to update") updateAccCmd.Flags().String("app", "photos", "Specify the app, default is 'photos'") accountCmd.AddCommand(listAccCmd, addAccCmd, updateAccCmd) diff --git a/pkg/account.go b/pkg/account.go index 7d5524f97..7ba782738 100644 --- a/pkg/account.go +++ b/pkg/account.go @@ -154,7 +154,7 @@ func (c *ClICtrl) UpdateAccount(ctx context.Context, params model.UpdateAccountP } } if acc == nil { - return fmt.Errorf("account not found") + return fmt.Errorf("account not found, use `account list` to list accounts") } if params.ExportDir != nil && *params.ExportDir != "" { _, err := internal.ValidateDirForWrite(*params.ExportDir) diff --git a/pkg/sync.go b/pkg/sync.go index f0b822d14..2fb0f6c01 100644 --- a/pkg/sync.go +++ b/pkg/sync.go @@ -2,6 +2,7 @@ package pkg import ( "cli-go/internal" + "cli-go/internal/api" "cli-go/pkg/model" "context" "encoding/base64" @@ -31,6 +32,10 @@ func (c *ClICtrl) Export() error { log.Printf("Skip export, error: %v while validing exportDir %s\n", err, account.ExportDir) continue } + if account.App == api.AppAuth { + log.Printf("Skip account %s: auth export is not supported", account.Email) + continue + } log.Println("start sync") err = c.SyncAccount(account) if err != nil { diff --git a/release.sh b/release.sh index f3886da41..fb74e3731 100755 --- a/release.sh +++ b/release.sh @@ -24,6 +24,10 @@ do if [ "$OS" == "darwin" ]; then BINARY_NAME="ente-cli-mac" fi + + if [ "$OS" == "linux" ]; then + BINARY_NAME="ente-cli-linux" + fi # make bin directory if it doesn't exist mkdir -p bin From 3e29d17625b117d2a674d95a3399b50cbee3577a Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 18 Oct 2023 13:55:29 +0530 Subject: [PATCH 113/183] Fix: Handle rename across volumes --- pkg/disk.go | 40 ++++++++++++++++++++++++++++++++++++++ pkg/remote_to_disk_file.go | 6 +++--- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/pkg/disk.go b/pkg/disk.go index adb9345fc..2a81d4976 100644 --- a/pkg/disk.go +++ b/pkg/disk.go @@ -5,7 +5,9 @@ import ( "cli-go/pkg/model/export" "encoding/json" "errors" + "io" "os" + "strings" ) const ( @@ -96,3 +98,41 @@ func readJSONFromFile(filePath string, data interface{}) error { decoder := json.NewDecoder(file) return decoder.Decode(data) } + +func Move(source, destination string) error { + err := os.Rename(source, destination) + if err != nil && strings.Contains(err.Error(), "cross-device link") { + return moveCrossDevice(source, destination) + } + return err +} + +func moveCrossDevice(source, destination string) error { + src, err := os.Open(source) + if err != nil { + return err + } + dst, err := os.Create(destination) + if err != nil { + src.Close() + return err + } + _, err = io.Copy(dst, src) + src.Close() + dst.Close() + if err != nil { + return err + } + fi, err := os.Stat(source) + if err != nil { + os.Remove(destination) + return err + } + err = os.Chmod(destination, fi.Mode()) + if err != nil { + os.Remove(destination) + return err + } + os.Remove(source) + return nil +} diff --git a/pkg/remote_to_disk_file.go b/pkg/remote_to_disk_file.go index 3dbe27bf7..a1c861414 100644 --- a/pkg/remote_to_disk_file.go +++ b/pkg/remote_to_disk_file.go @@ -145,11 +145,11 @@ func (c *ClICtrl) downloadEntry(ctx context.Context, imageFilePath := filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, imageFileName) videoFilePath := filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, videoFileName) // move the decrypt file to filePath - err = os.Rename(imagePath, imageFilePath) + err = Move(imagePath, imageFilePath) if err != nil { return err } - err = os.Rename(videoPath, videoFilePath) + err = Move(videoPath, videoFilePath) if err != nil { return err } @@ -159,7 +159,7 @@ func (c *ClICtrl) downloadEntry(ctx context.Context, fileName := baseFileName + extension filePath := filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, fileName) // move the decrypt file to filePath - err = os.Rename(*decrypt, filePath) + err = Move(*decrypt, filePath) if err != nil { return err } From 98585407d31ee853928d9e4a0f99dd344339102b Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 18 Oct 2023 17:52:41 +0530 Subject: [PATCH 114/183] Update documentation for docker --- README.md | 39 ++++++++++++++++++++---------- docker-compose.yml | 11 +++++++++ main.go | 14 ++++++++++- pkg/secrets/secret.go | 46 +++++++++++++++++++++++++++++++++++- utils/constants/constants.go | 3 +++ 5 files changed, 99 insertions(+), 14 deletions(-) create mode 100644 docker-compose.yml create mode 100644 utils/constants/constants.go diff --git a/README.md b/README.md index 641213f1f..5371a1790 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,33 @@ ente-cli export ``` -## Testing +## Docker + +### Configure +Modify the `docker-compose.yml` and add volume. +``cli-data`` volume is mandatory, you can add more volumes for your export directory. + * Build the docker image + ```shell + docker build -t ente-cli:latest . + ``` + * Start the container in detached mode + ```bash + docker-compose up -d + ``` +exec into the container +```shell + docker-compose exec ente-cli /bin/sh +``` + + +#### How to directly execute the command + + ```shell + docker run -it --rm ente-cli:latest ls + ``` + + +## Build locally Run the release script to build the binary and run it. @@ -44,14 +70,3 @@ or you can run the following command ```shell ./bin/ente-cli --help ``` - - -## Docker - Build the docker image - ```shell - docker build -t ente-cli:latest . - ``` - Run the commands using: - ```shell - docker run -it --rm ente-cli:latest ./ente-cli --help - ``` diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..b3c8ad0ce --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,11 @@ +version: '3' +services: + ente-cli: + image: ente-cli:latest + command: /bin/sh + volumes: + # Replace /Volumes/Data/ with a folder path on your system, typically $HOME/.ente-cli/ + - ~/.ente-cli/:/cli-data:rw +# - ~/Downloads/export-data:/data:rw + stdin_open: true + tty: true diff --git a/main.go b/main.go index 168fb921d..624dd45c8 100644 --- a/main.go +++ b/main.go @@ -2,13 +2,25 @@ package main import ( "cli-go/cmd" + "cli-go/internal" "cli-go/internal/api" "cli-go/pkg" "cli-go/pkg/secrets" + "cli-go/utils/constants" + "fmt" + "log" ) func main() { - db, err := pkg.GetDB("ente-cli.db") + cliDBPath := "" + if secrets.IsRunningInContainer() { + cliDBPath = constants.CliDataPath + _, err := internal.ValidateDirForWrite(cliDBPath) + if err != nil { + log.Fatalf("Please mount a volume to %s to persist cli data\n%v\n", cliDBPath, err) + } + } + db, err := pkg.GetDB(fmt.Sprintf("%sente-cli.db", cliDBPath)) if err != nil { panic(err) } diff --git a/pkg/secrets/secret.go b/pkg/secrets/secret.go index 3c67bc7b0..49b61a232 100644 --- a/pkg/secrets/secret.go +++ b/pkg/secrets/secret.go @@ -1,20 +1,32 @@ package secrets import ( + "cli-go/utils/constants" "crypto/rand" "errors" "fmt" "log" + "os" "github.com/zalando/go-keyring" ) +func IsRunningInContainer() bool { + if _, err := os.Stat("/.dockerenv"); err != nil { + return false + } + return true +} func GetOrCreateClISecret() []byte { // get password secret, err := keyring.Get("ente-cli-cli", "ghost") if err != nil { if !errors.Is(err, keyring.ErrNotFound) { - log.Fatal(fmt.Errorf("error getting password from keyring: %w", err)) + if IsRunningInContainer() { + return GetSecretFromSecretText() + } else { + log.Fatal(fmt.Errorf("error getting password from keyring: %w", err)) + } } key := make([]byte, 32) _, err = rand.Read(key) @@ -30,3 +42,35 @@ func GetOrCreateClISecret() []byte { } return []byte(secret) } + +// GetSecretFromSecretText reads the scecret from the secret text file. +// If the file does not exist, it will be created and write random 32 byte secret to it. +func GetSecretFromSecretText() []byte { + // Define the path to the secret text file + secretFilePath := fmt.Sprintf("%s.secret.txt", constants.CliDataPath) + + // Check if file exists + _, err := os.Stat(secretFilePath) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + log.Fatal(fmt.Errorf("error checking secret file: %w", err)) + } + // File does not exist; create and write a random 32-byte secret + key := make([]byte, 32) + _, err := rand.Read(key) + if err != nil { + log.Fatal(fmt.Errorf("error generating key: %w", err)) + } + err = os.WriteFile(secretFilePath, key, 0644) + if err != nil { + log.Fatal(fmt.Errorf("error writing to secret file: %w", err)) + } + return key + } + // File exists; read the secret + secret, err := os.ReadFile(secretFilePath) + if err != nil { + log.Fatal(fmt.Errorf("error reading from secret file: %w", err)) + } + return secret +} diff --git a/utils/constants/constants.go b/utils/constants/constants.go new file mode 100644 index 000000000..7209d6466 --- /dev/null +++ b/utils/constants/constants.go @@ -0,0 +1,3 @@ +package constants + +const CliDataPath = "/cli-data/" From b7a10bb5ffd3d419a88697eab5d9f3da47a7c265 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 18 Oct 2023 17:53:43 +0530 Subject: [PATCH 115/183] Add Dockerfile-x86 for X86 --- Dockerfile-x86 | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 Dockerfile-x86 diff --git a/Dockerfile-x86 b/Dockerfile-x86 new file mode 100644 index 000000000..23a877cb9 --- /dev/null +++ b/Dockerfile-x86 @@ -0,0 +1,20 @@ +FROM golang:1.20-alpine3.17@sha256:9c2f89db6fda13c3c480749787f62fed5831699bb2c32881b8f327f1cf7bae42 as builder386 +RUN apt-get update +RUN apt-get install -y gcc +RUN apt-get install -y git +RUN apt-get install -y pkg-config +RUN apt-get install -y libsodium-dev + + +ENV GOOS=linux + +WORKDIR /etc/ente/ +RUN uname -a +COPY go.mod . +COPY go.sum . +RUN go mod download + + +COPY . . +RUN --mount=type=cache,target=/root/.cache/go-build \ + go build -o ente-cli main.go From ccb8a5834c1689f21fcd27edafe576f70c5b58b5 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 20 Oct 2023 10:17:02 +0530 Subject: [PATCH 116/183] Switch to blake2b-simd for kdf --- go.mod | 5 +++++ go.sum | 12 ++++++++++++ internal/crypto/crypto.go | 41 ++++++++++++++++++++++++++++++++++++--- 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 17f39f5a7..52128ede9 100644 --- a/go.mod +++ b/go.mod @@ -10,12 +10,17 @@ require ( ) require ( + git.schwanenlied.me/yawning/chacha20.git v0.0.0-20170904085104-e3b1f968fc63 // indirect + github.com/Yawning/chacha20 v0.0.0-20170904085104-e3b1f968fc63 // indirect + github.com/Yawning/poly1305 v0.0.0-20151107134637-dfc796fe731c // indirect github.com/alessio/shellescape v1.4.1 // indirect github.com/danieljoos/wincred v1.2.0 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect + github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect github.com/zalando/go-keyring v0.2.3 // indirect + go.artemisc.eu/godium v0.0.0-20201014025004-f08c318a71b3 // indirect ) require ( diff --git a/go.sum b/go.sum index 978c21713..2614752b4 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,14 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +git.schwanenlied.me/yawning/chacha20.git v0.0.0-20170904085104-e3b1f968fc63 h1:bwZNsbw3qFbg6ox55HrA37nPmh+/wtJxZ7uWeiAdUUc= +git.schwanenlied.me/yawning/chacha20.git v0.0.0-20170904085104-e3b1f968fc63/go.mod h1:NYi4Ifd1g/YbhIDgDfw6t7QdsW4tofQWMX/+FiDtJWs= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Yawning/chacha20 v0.0.0-20170904085104-e3b1f968fc63 h1:I6/SJSN9wJMJ+ZyQaCHUlzoTA4ypU5Bb44YWR1wTY/0= +github.com/Yawning/chacha20 v0.0.0-20170904085104-e3b1f968fc63/go.mod h1:nf+Komq6fVP4SwmKEaVGxHTyQGKREVlwjQKpvOV39yE= +github.com/Yawning/poly1305 v0.0.0-20151107134637-dfc796fe731c h1:JD5KufmlwGs/wsQhS1HrKKDMXYhLmp4XQvgoAoua4A0= +github.com/Yawning/poly1305 v0.0.0-20151107134637-dfc796fe731c/go.mod h1:CkwFWTKoa4/jtX6RLTogyqTlMn4898oEkv6j2Ai50+I= github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -53,6 +59,7 @@ github.com/danieljoos/wincred v1.2.0 h1:ozqKHaLK0W/ii4KVbbvluM91W2H3Sh0BncbUNPS7 github.com/danieljoos/wincred v1.2.0/go.mod h1:FzQLLMKBFdvu+osBrnFODiv32YGwCfx0SkRa/eYHgec= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dchest/siphash v1.2.2/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -152,6 +159,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= @@ -192,6 +201,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/zalando/go-keyring v0.2.3 h1:v9CUu9phlABObO4LPWycf+zwMG7nlbb3t/B5wa97yms= github.com/zalando/go-keyring v0.2.3/go.mod h1:HL4k+OXQfJUWaMnqyuSOc0drfGPX2b51Du6K+MRgZMk= +go.artemisc.eu/godium v0.0.0-20201014025004-f08c318a71b3 h1:UB0aMKbtlmdvZ3W1C3U3wFnXcS8tNTNksXF968gB3tc= +go.artemisc.eu/godium v0.0.0-20201014025004-f08c318a71b3/go.mod h1:Pav9C606SsOk8r0N+Ow8PQwA+C9WontLIGzfIFgPj9A= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -205,6 +216,7 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go index 3f7e80f94..ce5f4c2fd 100644 --- a/internal/crypto/crypto.go +++ b/internal/crypto/crypto.go @@ -5,7 +5,9 @@ import ( "bytes" "cli-go/utils/encoding" "encoding/base64" + "encoding/binary" "fmt" + blake2b "github.com/minio/blake2b-simd" "io" "log" "os" @@ -123,15 +125,48 @@ func EncryptChaCha20poly1305(data []byte, key []byte) ([]byte, []byte, error) { // This loginKey act as user provided password during SRP authentication. // Parameters: keyEncKey: This is the keyEncryptionKey that is derived from the user's password. func DeriveLoginKey(keyEncKey []byte) []byte { - mainKey := sodium.MasterKey{Bytes: keyEncKey} - subKey := mainKey.Derive(loginSubKeyLen, loginSubKeyId, loginSubKeyContext).Bytes + subKey, _ := deriveSubKey(keyEncKey, loginSubKeyContext, loginSubKeyId, loginSubKeyLen) // return the first 16 bytes of the derived key return subKey[:16] } +const ( + cryptoKDFBlake2bBytesMin = 16 + cryptoKDFBlake2bBytesMax = 64 + cryptoGenerichashBlake2bSaltBytes = 16 + cryptoGenerichashBlake2bPersonalBytes = 16 +) + +func deriveSubKey(masterKey []byte, context string, subKeyID uint64, subKeyLength uint32) ([]byte, error) { + if subKeyLength < cryptoKDFBlake2bBytesMin || subKeyLength > cryptoKDFBlake2bBytesMax { + return nil, fmt.Errorf("subKeyLength out of bounds") + } + // Pad the context + ctxPadded := make([]byte, cryptoGenerichashBlake2bPersonalBytes) + copy(ctxPadded, []byte(context)) + // Convert subKeyID to byte slice and pad + salt := make([]byte, cryptoGenerichashBlake2bSaltBytes) + binary.LittleEndian.PutUint64(salt, subKeyID) + + // Create a BLAKE2b configuration + config := &blake2b.Config{ + Size: uint8(subKeyLength), + Key: masterKey, + Salt: salt, + Person: ctxPadded, + } + hasher, err := blake2b.New(config) + if err != nil { + return nil, err + } + hasher.Write(nil) // No data, just using key, salt, and personalization + return hasher.Sum(nil), nil +} + func SecretBoxOpen(c []byte, n []byte, k []byte) ([]byte, error) { var cp sodium.Bytes = c - return cp.SecretBoxOpen(sodium.SecretBoxNonce{Bytes: n}, sodium.SecretBoxKey{Bytes: k}) + res, err := cp.SecretBoxOpen(sodium.SecretBoxNonce{Bytes: n}, sodium.SecretBoxKey{Bytes: k}) + return res, err } func SecretBoxOpenBase64(cipher string, nonce string, k []byte) ([]byte, error) { From 5f029fc12f1c7eeefe735fab62bfe9c3f3cad29b Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 20 Oct 2023 11:34:07 +0530 Subject: [PATCH 117/183] Minor refactor --- go.mod | 9 +- go.sum | 7 ++ internal/crypto/crypto.go | 170 ++-------------------------- internal/crypto/crypto_libsodium.go | 161 ++++++++++++++++++++++++++ internal/crypto/crypto_test.go | 2 +- pkg/model/enc_string.go | 2 +- 6 files changed, 182 insertions(+), 169 deletions(-) create mode 100644 internal/crypto/crypto_libsodium.go diff --git a/go.mod b/go.mod index 52128ede9..e1258a425 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,9 @@ require ( github.com/go-resty/resty/v2 v2.7.0 github.com/google/uuid v1.3.1 github.com/jamesruan/sodium v1.0.14 - golang.org/x/crypto v0.13.0 + github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 + github.com/pkg/errors v0.9.1 + golang.org/x/crypto v0.14.0 ) require ( @@ -18,7 +20,6 @@ require ( github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect - github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect github.com/zalando/go-keyring v0.2.3 // indirect go.artemisc.eu/godium v0.0.0-20201014025004-f08c318a71b3 // indirect ) @@ -41,8 +42,8 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect go.etcd.io/bbolt v1.3.7 // indirect golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.12.0 // indirect - golang.org/x/term v0.12.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 2614752b4..10f11fe85 100644 --- a/go.sum +++ b/go.sum @@ -165,6 +165,7 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -223,6 +224,8 @@ golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -351,9 +354,13 @@ golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go index ce5f4c2fd..65a9e9389 100644 --- a/internal/crypto/crypto.go +++ b/internal/crypto/crypto.go @@ -1,18 +1,10 @@ package crypto import ( - "bufio" - "bytes" - "cli-go/utils/encoding" "encoding/base64" "encoding/binary" "fmt" - blake2b "github.com/minio/blake2b-simd" - "io" - "log" - "os" - - "github.com/jamesruan/sodium" + "github.com/minio/blake2b-simd" "golang.org/x/crypto/argon2" ) @@ -23,6 +15,12 @@ const ( decryptionBufferSize = 4 * 1024 * 1024 ) +const ( + cryptoKDFBlake2bBytesMin = 16 + cryptoKDFBlake2bBytesMax = 64 + cryptoGenerichashBlake2bSaltBytes = 16 + cryptoGenerichashBlake2bPersonalBytes = 16 +) // DeriveArgonKey generates a 32-bit cryptographic key using the Argon2id algorithm. // Parameters: @@ -52,75 +50,6 @@ func DeriveArgonKey(password, salt string, memLimit, opsLimit int) ([]byte, erro return key, nil } -// DecryptChaCha20poly1305 decrypts the given data using the ChaCha20-Poly1305 algorithm. -// Parameters: -// - data: The encrypted data as a byte slice. -// - key: The key for decryption as a byte slice. -// - nonce: The nonce for decryption as a byte slice. -// -// Returns: -// - A byte slice representing the decrypted data. -// - An error object, which is nil if no error occurs. -func DecryptChaCha20poly1305(data []byte, key []byte, nonce []byte) ([]byte, error) { - reader := bytes.NewReader(data) - header := sodium.SecretStreamXCPHeader{Bytes: nonce} - decoder, err := sodium.MakeSecretStreamXCPDecoder( - sodium.SecretStreamXCPKey{Bytes: key}, - reader, - header) - if err != nil { - log.Println("Failed to make secret stream decoder", err) - return nil, err - } - // Buffer to store the decrypted data - decryptedData := make([]byte, len(data)) - n, err := decoder.Read(decryptedData) - if err != nil && err != io.EOF { - log.Println("Failed to read from decoder", err) - return nil, err - } - return decryptedData[:n], nil -} - -func DecryptChaChaBase64(data string, key []byte, nonce string) (string, []byte, error) { - // Decode data from base64 - dataBytes, err := base64.StdEncoding.DecodeString(data) - if err != nil { - return "", nil, fmt.Errorf("invalid data: %v", err) - } - // Decode nonce from base64 - nonceBytes, err := base64.StdEncoding.DecodeString(nonce) - if err != nil { - return "", nil, fmt.Errorf("invalid nonce: %v", err) - } - // Decrypt data - decryptedData, err := DecryptChaCha20poly1305(dataBytes, key, nonceBytes) - if err != nil { - return "", nil, fmt.Errorf("failed to decrypt data: %v", err) - } - return base64.StdEncoding.EncodeToString(decryptedData), decryptedData, nil -} - -// EncryptChaCha20poly1305 encrypts the given data using the ChaCha20-Poly1305 algorithm. -// Parameters: -// - data: The plaintext data as a byte slice. -// - key: The key for encryption as a byte slice. -// -// Returns: -// - A byte slice representing the encrypted data. -// - A byte slice representing the header of the encrypted data. -// - An error object, which is nil if no error occurs. -func EncryptChaCha20poly1305(data []byte, key []byte) ([]byte, []byte, error) { - var buf bytes.Buffer - encoder := sodium.MakeSecretStreamXCPEncoder(sodium.SecretStreamXCPKey{Bytes: key}, &buf) - _, err := encoder.WriteAndClose(data) - if err != nil { - log.Println("Failed to write to encoder", err) - return nil, nil, err - } - return buf.Bytes(), encoder.Header().Bytes, nil -} - // DeriveLoginKey derives a login key from the given key encryption key. // This loginKey act as user provided password during SRP authentication. // Parameters: keyEncKey: This is the keyEncryptionKey that is derived from the user's password. @@ -130,13 +59,6 @@ func DeriveLoginKey(keyEncKey []byte) []byte { return subKey[:16] } -const ( - cryptoKDFBlake2bBytesMin = 16 - cryptoKDFBlake2bBytesMax = 64 - cryptoGenerichashBlake2bSaltBytes = 16 - cryptoGenerichashBlake2bPersonalBytes = 16 -) - func deriveSubKey(masterKey []byte, context string, subKeyID uint64, subKeyLength uint32) ([]byte, error) { if subKeyLength < cryptoKDFBlake2bBytesMin || subKeyLength > cryptoKDFBlake2bBytesMax { return nil, fmt.Errorf("subKeyLength out of bounds") @@ -162,81 +84,3 @@ func deriveSubKey(masterKey []byte, context string, subKeyID uint64, subKeyLengt hasher.Write(nil) // No data, just using key, salt, and personalization return hasher.Sum(nil), nil } - -func SecretBoxOpen(c []byte, n []byte, k []byte) ([]byte, error) { - var cp sodium.Bytes = c - res, err := cp.SecretBoxOpen(sodium.SecretBoxNonce{Bytes: n}, sodium.SecretBoxKey{Bytes: k}) - return res, err -} - -func SecretBoxOpenBase64(cipher string, nonce string, k []byte) ([]byte, error) { - var cp sodium.Bytes = encoding.DecodeBase64(cipher) - out, err := cp.SecretBoxOpen(sodium.SecretBoxNonce{Bytes: encoding.DecodeBase64(nonce)}, sodium.SecretBoxKey{Bytes: k}) - if err != nil { - return nil, err - } - return out, nil -} - -func SealedBoxOpen(cipherText []byte, publicKey, masterSecret []byte) ([]byte, error) { - var cp sodium.Bytes = cipherText - om, err := cp.SealedBoxOpen(sodium.BoxKP{ - PublicKey: sodium.BoxPublicKey{Bytes: publicKey}, - SecretKey: sodium.BoxSecretKey{Bytes: masterSecret}, - }) - if err != nil { - return nil, fmt.Errorf("failed to open sealed box: %v", err) - } - return om, nil -} - -func DecryptFile(encryptedFilePath string, decryptedFilePath string, key, nonce []byte) error { - inputFile, err := os.Open(encryptedFilePath) - if err != nil { - return err - } - defer inputFile.Close() - - outputFile, err := os.Create(decryptedFilePath) - if err != nil { - return err - } - defer outputFile.Close() - - reader := bufio.NewReader(inputFile) - writer := bufio.NewWriter(outputFile) - - header := sodium.SecretStreamXCPHeader{Bytes: nonce} - decoder, err := sodium.MakeSecretStreamXCPDecoder( - sodium.SecretStreamXCPKey{Bytes: key}, - reader, - header) - if err != nil { - log.Println("Failed to make secret stream decoder", err) - return err - } - - buf := make([]byte, decryptionBufferSize) - for { - n, errErr := decoder.Read(buf) - if errErr != nil && errErr != io.EOF { - log.Println("Failed to read from decoder", errErr) - return errErr - } - if n == 0 { - break - } - if _, err := writer.Write(buf[:n]); err != nil { - log.Println("Failed to write to output file", err) - return err - } - if errErr == io.EOF { - break - } - } - if err := writer.Flush(); err != nil { - log.Println("Failed to flush writer", err) - return err - } - return nil -} diff --git a/internal/crypto/crypto_libsodium.go b/internal/crypto/crypto_libsodium.go new file mode 100644 index 000000000..c689946d2 --- /dev/null +++ b/internal/crypto/crypto_libsodium.go @@ -0,0 +1,161 @@ +package crypto + +import ( + "bufio" + "bytes" + "cli-go/utils/encoding" + "encoding/base64" + "fmt" + "github.com/jamesruan/sodium" + "io" + "log" + "os" +) + +// EncryptChaCha20poly1305 encrypts the given data using the ChaCha20-Poly1305 algorithm. +// Parameters: +// - data: The plaintext data as a byte slice. +// - key: The key for encryption as a byte slice. +// +// Returns: +// - A byte slice representing the encrypted data. +// - A byte slice representing the header of the encrypted data. +// - An error object, which is nil if no error occurs. +func EncryptChaCha20poly1305(data []byte, key []byte) ([]byte, []byte, error) { + var buf bytes.Buffer + encoder := sodium.MakeSecretStreamXCPEncoder(sodium.SecretStreamXCPKey{Bytes: key}, &buf) + _, err := encoder.WriteAndClose(data) + if err != nil { + log.Println("Failed to write to encoder", err) + return nil, nil, err + } + return buf.Bytes(), encoder.Header().Bytes, nil +} + +// decryptChaCha20poly1305 decrypts the given data using the ChaCha20-Poly1305 algorithm. +// Parameters: +// - data: The encrypted data as a byte slice. +// - key: The key for decryption as a byte slice. +// - nonce: The nonce for decryption as a byte slice. +// +// Returns: +// - A byte slice representing the decrypted data. +// - An error object, which is nil if no error occurs. + +func decryptChaCha20poly1305(data []byte, key []byte, nonce []byte) ([]byte, error) { + reader := bytes.NewReader(data) + header := sodium.SecretStreamXCPHeader{Bytes: nonce} + decoder, err := sodium.MakeSecretStreamXCPDecoder( + sodium.SecretStreamXCPKey{Bytes: key}, + reader, + header) + if err != nil { + log.Println("Failed to make secret stream decoder", err) + return nil, err + } + // Buffer to store the decrypted data + decryptedData := make([]byte, len(data)) + n, err := decoder.Read(decryptedData) + if err != nil && err != io.EOF { + log.Println("Failed to read from decoder", err) + return nil, err + } + return decryptedData[:n], nil +} + +func DecryptChaChaBase64(data string, key []byte, nonce string) (string, []byte, error) { + // Decode data from base64 + dataBytes, err := base64.StdEncoding.DecodeString(data) + if err != nil { + return "", nil, fmt.Errorf("invalid data: %v", err) + } + // Decode nonce from base64 + nonceBytes, err := base64.StdEncoding.DecodeString(nonce) + if err != nil { + return "", nil, fmt.Errorf("invalid nonce: %v", err) + } + // Decrypt data + decryptedData, err := decryptChaCha20poly1305(dataBytes, key, nonceBytes) + if err != nil { + return "", nil, fmt.Errorf("failed to decrypt data: %v", err) + } + return base64.StdEncoding.EncodeToString(decryptedData), decryptedData, nil +} + +func SecretBoxOpen(c []byte, n []byte, k []byte) ([]byte, error) { + var cp sodium.Bytes = c + res, err := cp.SecretBoxOpen(sodium.SecretBoxNonce{Bytes: n}, sodium.SecretBoxKey{Bytes: k}) + return res, err +} + +func SecretBoxOpenBase64(cipher string, nonce string, k []byte) ([]byte, error) { + var cp sodium.Bytes = encoding.DecodeBase64(cipher) + out, err := cp.SecretBoxOpen(sodium.SecretBoxNonce{Bytes: encoding.DecodeBase64(nonce)}, sodium.SecretBoxKey{Bytes: k}) + if err != nil { + return nil, err + } + return out, nil +} + +func SealedBoxOpen(cipherText []byte, publicKey, masterSecret []byte) ([]byte, error) { + var cp sodium.Bytes = cipherText + om, err := cp.SealedBoxOpen(sodium.BoxKP{ + PublicKey: sodium.BoxPublicKey{Bytes: publicKey}, + SecretKey: sodium.BoxSecretKey{Bytes: masterSecret}, + }) + if err != nil { + return nil, fmt.Errorf("failed to open sealed box: %v", err) + } + return om, nil +} + +func DecryptFile(encryptedFilePath string, decryptedFilePath string, key, nonce []byte) error { + inputFile, err := os.Open(encryptedFilePath) + if err != nil { + return err + } + defer inputFile.Close() + + outputFile, err := os.Create(decryptedFilePath) + if err != nil { + return err + } + defer outputFile.Close() + + reader := bufio.NewReader(inputFile) + writer := bufio.NewWriter(outputFile) + + header := sodium.SecretStreamXCPHeader{Bytes: nonce} + decoder, err := sodium.MakeSecretStreamXCPDecoder( + sodium.SecretStreamXCPKey{Bytes: key}, + reader, + header) + if err != nil { + log.Println("Failed to make secret stream decoder", err) + return err + } + + buf := make([]byte, decryptionBufferSize) + for { + n, errErr := decoder.Read(buf) + if errErr != nil && errErr != io.EOF { + log.Println("Failed to read from decoder", errErr) + return errErr + } + if n == 0 { + break + } + if _, err := writer.Write(buf[:n]); err != nil { + log.Println("Failed to write to output file", err) + return err + } + if errErr == io.EOF { + break + } + } + if err := writer.Flush(); err != nil { + log.Println("Failed to flush writer", err) + return err + } + return nil +} diff --git a/internal/crypto/crypto_test.go b/internal/crypto/crypto_test.go index 98fd26890..9f3c60bea 100644 --- a/internal/crypto/crypto_test.go +++ b/internal/crypto/crypto_test.go @@ -43,7 +43,7 @@ func TestDecryptChaCha20poly1305(t *testing.T) { t.Fatalf("Failed to decode cipher nonce: %v", err) } - decryptedText, err := DecryptChaCha20poly1305(decodedCipherText, derivedKey, decodedCipherNonce) + decryptedText, err := decryptChaCha20poly1305(decodedCipherText, derivedKey, decodedCipherNonce) if err != nil { t.Fatalf("Failed to decrypt: %v", err) } diff --git a/pkg/model/enc_string.go b/pkg/model/enc_string.go index 09f480860..ebbbb891a 100644 --- a/pkg/model/enc_string.go +++ b/pkg/model/enc_string.go @@ -23,7 +23,7 @@ func MakeEncString(plainTextBytes []byte, key []byte) *EncString { } func (e *EncString) MustDecrypt(key []byte) []byte { - plainBytes, err := crypto.DecryptChaCha20poly1305(encoding.DecodeBase64(e.CipherText), key, encoding.DecodeBase64(e.Nonce)) + _, plainBytes, err := crypto.DecryptChaChaBase64(e.CipherText, key, e.Nonce) if err != nil { panic(err) } From 2d8b3eb2158fa80d104163273e76d8fb6902e1c3 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 20 Oct 2023 13:20:49 +0530 Subject: [PATCH 118/183] Add decryptChacha using native go --- internal/crypto/crypto.go | 19 ++ internal/crypto/crypto_libsodium.go | 30 +-- internal/crypto/stream.go | 397 ++++++++++++++++++++++++++++ 3 files changed, 424 insertions(+), 22 deletions(-) create mode 100644 internal/crypto/stream.go diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go index 65a9e9389..b757c996b 100644 --- a/internal/crypto/crypto.go +++ b/internal/crypto/crypto.go @@ -84,3 +84,22 @@ func deriveSubKey(masterKey []byte, context string, subKeyID uint64, subKeyLengt hasher.Write(nil) // No data, just using key, salt, and personalization return hasher.Sum(nil), nil } + +func DecryptChaChaBase64(data string, key []byte, nonce string) (string, []byte, error) { + // Decode data from base64 + dataBytes, err := base64.StdEncoding.DecodeString(data) + if err != nil { + return "", nil, fmt.Errorf("invalid data: %v", err) + } + // Decode nonce from base64 + nonceBytes, err := base64.StdEncoding.DecodeString(nonce) + if err != nil { + return "", nil, fmt.Errorf("invalid nonce: %v", err) + } + // Decrypt data + decryptedData, err := decryptChaCha20poly1305(dataBytes, key, nonceBytes) + if err != nil { + return "", nil, fmt.Errorf("failed to decrypt data: %v", err) + } + return base64.StdEncoding.EncodeToString(decryptedData), decryptedData, nil +} diff --git a/internal/crypto/crypto_libsodium.go b/internal/crypto/crypto_libsodium.go index c689946d2..04b87844c 100644 --- a/internal/crypto/crypto_libsodium.go +++ b/internal/crypto/crypto_libsodium.go @@ -4,7 +4,6 @@ import ( "bufio" "bytes" "cli-go/utils/encoding" - "encoding/base64" "fmt" "github.com/jamesruan/sodium" "io" @@ -41,8 +40,7 @@ func EncryptChaCha20poly1305(data []byte, key []byte) ([]byte, []byte, error) { // Returns: // - A byte slice representing the decrypted data. // - An error object, which is nil if no error occurs. - -func decryptChaCha20poly1305(data []byte, key []byte, nonce []byte) ([]byte, error) { +func decryptChaCha20poly1305LibSodium(data []byte, key []byte, nonce []byte) ([]byte, error) { reader := bytes.NewReader(data) header := sodium.SecretStreamXCPHeader{Bytes: nonce} decoder, err := sodium.MakeSecretStreamXCPDecoder( @@ -63,23 +61,16 @@ func decryptChaCha20poly1305(data []byte, key []byte, nonce []byte) ([]byte, err return decryptedData[:n], nil } -func DecryptChaChaBase64(data string, key []byte, nonce string) (string, []byte, error) { - // Decode data from base64 - dataBytes, err := base64.StdEncoding.DecodeString(data) +func decryptChaCha20poly1305(data []byte, key []byte, nonce []byte) ([]byte, error) { + decryptor, err := NewDecryptor(key, nonce) if err != nil { - return "", nil, fmt.Errorf("invalid data: %v", err) + return nil, err } - // Decode nonce from base64 - nonceBytes, err := base64.StdEncoding.DecodeString(nonce) + decoded, _, err := decryptor.Pull(data) if err != nil { - return "", nil, fmt.Errorf("invalid nonce: %v", err) + return nil, err } - // Decrypt data - decryptedData, err := decryptChaCha20poly1305(dataBytes, key, nonceBytes) - if err != nil { - return "", nil, fmt.Errorf("failed to decrypt data: %v", err) - } - return base64.StdEncoding.EncodeToString(decryptedData), decryptedData, nil + return decoded, nil } func SecretBoxOpen(c []byte, n []byte, k []byte) ([]byte, error) { @@ -89,12 +80,7 @@ func SecretBoxOpen(c []byte, n []byte, k []byte) ([]byte, error) { } func SecretBoxOpenBase64(cipher string, nonce string, k []byte) ([]byte, error) { - var cp sodium.Bytes = encoding.DecodeBase64(cipher) - out, err := cp.SecretBoxOpen(sodium.SecretBoxNonce{Bytes: encoding.DecodeBase64(nonce)}, sodium.SecretBoxKey{Bytes: k}) - if err != nil { - return nil, err - } - return out, nil + return SecretBoxOpen(encoding.DecodeBase64(cipher), encoding.DecodeBase64(nonce), k) } func SealedBoxOpen(cipherText []byte, publicKey, masterSecret []byte) ([]byte, error) { diff --git a/internal/crypto/stream.go b/internal/crypto/stream.go new file mode 100644 index 000000000..6d4359b16 --- /dev/null +++ b/internal/crypto/stream.go @@ -0,0 +1,397 @@ +package crypto + +import ( + "bytes" + "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "golang.org/x/crypto/chacha20" + "golang.org/x/crypto/chacha20poly1305" + "golang.org/x/crypto/poly1305" +) + +// public constants +const ( + TagMessage = 0 + TagPush = 0x01 + TagRekey = 0x02 + TagFinal = TagPush | TagRekey + + StreamKeyBytes = chacha20poly1305.KeySize + StreamHeaderBytes = chacha20poly1305.NonceSizeX + StreamABytes = 16 + 1 +) + +const crypto_core_hchacha20_INPUTBYTES = 16 + +/* const crypto_secretstream_xchacha20poly1305_INONCEBYTES = 8 */ +const crypto_secretstream_xchacha20poly1305_COUNTERBYTES = 4 + +var pad0 [16]byte + +var invalidKey = errors.New("invalid key") +var invalidInput = errors.New("invalid input") +var cryptoFailure = errors.New("crypto failed") + +type streamState struct { + k [StreamKeyBytes]byte + nonce [chacha20poly1305.NonceSize]byte + pad [8]byte +} + +func (s *streamState) reset() { + for i := range s.nonce { + s.nonce[i] = 0 + } + s.nonce[0] = 1 +} + +type Encryptor interface { + Push(m []byte, tag byte) ([]byte, error) +} + +type Decryptor interface { + Pull(m []byte) ([]byte, byte, error) +} + +type encryptor struct { + streamState +} + +type decryptor struct { + streamState +} + +func NewStreamKey() []byte { + k := make([]byte, chacha20poly1305.KeySize) + _, _ = rand.Read(k) + return k +} + +func NewEncryptor(key []byte) (Encryptor, []byte, error) { + if len(key) != StreamKeyBytes { + return nil, nil, invalidKey + } + + header := make([]byte, StreamHeaderBytes) + _, _ = rand.Read(header) + + stream := &encryptor{} + + k, err := chacha20.HChaCha20(key[:], header[:16]) + if err != nil { + //fmt.Printf("error: %v", err) + return nil, nil, err + } + copy(stream.k[:], k) + stream.reset() + + for i := range stream.pad { + stream.pad[i] = 0 + } + + for i, b := range header[crypto_core_hchacha20_INPUTBYTES:] { + stream.nonce[i+crypto_secretstream_xchacha20poly1305_COUNTERBYTES] = b + } + // fmt.Printf("stream: %+v\n", stream.streamState) + + return stream, header, nil +} + +func (s *encryptor) Push(plain []byte, tag byte) ([]byte, error) { + var err error + + //crypto_onetimeauth_poly1305_state poly1305_state; + var poly *poly1305.MAC + + //unsigned char block[64U]; + var block [64]byte + + //unsigned char slen[8U]; + var slen [8]byte + + //unsigned char *c; + //unsigned char *mac; + // + //if (outlen_p != NULL) { + //*outlen_p = 0U; + //} + + mlen := len(plain) + //if (mlen > crypto_secretstream_xchacha20poly1305_MESSAGEBYTES_MAX) { + //sodium_misuse(); + //} + + out := make([]byte, mlen+StreamABytes) + + chacha, err := chacha20.NewUnauthenticatedCipher(s.k[:], s.nonce[:]) + if err != nil { + return nil, err + } + //crypto_stream_chacha20_ietf(block, sizeof block, state->nonce, state->k); + chacha.XORKeyStream(block[:], block[:]) + + //crypto_onetimeauth_poly1305_init(&poly1305_state, block); + var poly_init [32]byte + copy(poly_init[:], block[:]) + poly = poly1305.New(&poly_init) + + // TODO add support for add data + //sodium_memzero(block, sizeof block); + //crypto_onetimeauth_poly1305_update(&poly1305_state, ad, adlen); + //crypto_onetimeauth_poly1305_update(&poly1305_state, _pad0, + //(0x10 - adlen) & 0xf); + + //memset(block, 0, sizeof block); + //block[0] = tag; + memzero(block[:]) + block[0] = tag + + // + //crypto_stream_chacha20_ietf_xor_ic(block, block, sizeof block, state->nonce, 1U, state->k); + //crypto_onetimeauth_poly1305_update(&poly1305_state, block, sizeof block); + //out[0] = block[0]; + chacha.XORKeyStream(block[:], block[:]) + _, _ = poly.Write(block[:]) + out[0] = block[0] + + // + //c = out + (sizeof tag); + c := out[1:] + //crypto_stream_chacha20_ietf_xor_ic(c, m, mlen, state->nonce, 2U, state->k); + //crypto_onetimeauth_poly1305_update(&poly1305_state, c, mlen); + //crypto_onetimeauth_poly1305_update (&poly1305_state, _pad0, (0x10 - (sizeof block) + mlen) & 0xf); + chacha.XORKeyStream(c, plain) + _, _ = poly.Write(c[:mlen]) + padlen := (0x10 - len(block) + mlen) & 0xf + _, _ = poly.Write(pad0[:padlen]) + + // + //STORE64_LE(slen, (uint64_t) adlen); + //crypto_onetimeauth_poly1305_update(&poly1305_state, slen, sizeof slen); + binary.LittleEndian.PutUint64(slen[:], uint64(0)) + _, _ = poly.Write(slen[:]) + + //STORE64_LE(slen, (sizeof block) + mlen); + //crypto_onetimeauth_poly1305_update(&poly1305_state, slen, sizeof slen); + binary.LittleEndian.PutUint64(slen[:], uint64(len(block)+mlen)) + _, _ = poly.Write(slen[:]) + + // + //mac = c + mlen; + //crypto_onetimeauth_poly1305_final(&poly1305_state, mac); + mac := c[mlen:] + copy(mac, poly.Sum(nil)) + //sodium_memzero(&poly1305_state, sizeof poly1305_state); + // + + //XOR_BUF(STATE_INONCE(state), mac, crypto_secretstream_xchacha20poly1305_INONCEBYTES); + //sodium_increment(STATE_COUNTER(state), crypto_secretstream_xchacha20poly1305_COUNTERBYTES); + xor_buf(s.nonce[crypto_secretstream_xchacha20poly1305_COUNTERBYTES:], mac) + buf_inc(s.nonce[:crypto_secretstream_xchacha20poly1305_COUNTERBYTES]) + + // TODO + //if ((tag & crypto_secretstream_xchacha20poly1305_TAG_REKEY) != 0 || + //sodium_is_zero(STATE_COUNTER(state), + //crypto_secretstream_xchacha20poly1305_COUNTERBYTES)) { + //crypto_secretstream_xchacha20poly1305_rekey(state); + //} + + //if (outlen_p != NULL) { + //*outlen_p = crypto_secretstream_xchacha20poly1305_ABYTES + mlen; + //} + + //return 0; + return out, nil +} + +func NewDecryptor(key, header []byte) (Decryptor, error) { + stream := &decryptor{} + + //crypto_core_hchacha20(state->k, in, k, NULL); + k, err := chacha20.HChaCha20(key, header[:16]) + if err != nil { + fmt.Printf("error: %v", err) + return nil, err + } + copy(stream.k[:], k) + + //_crypto_secretstream_xchacha20poly1305_counter_reset(state); + stream.reset() + + //memcpy(STATE_INONCE(state), in + crypto_core_hchacha20_INPUTBYTES, + // crypto_secretstream_xchacha20poly1305_INONCEBYTES); + copy(stream.nonce[crypto_secretstream_xchacha20poly1305_COUNTERBYTES:], + header[crypto_core_hchacha20_INPUTBYTES:]) + + //memset(state->_pad, 0, sizeof state->_pad); + copy(stream.pad[:], pad0[:]) + + //fmt.Printf("decryptor: %+v\n", stream.streamState) + + return stream, nil +} + +func (s *decryptor) Pull(in []byte) ([]byte, byte, error) { + inlen := len(in) + //crypto_onetimeauth_poly1305_state poly1305_state; + + //unsigned char block[64U]; + var block [64]byte + + //unsigned char slen[8U]; + var slen [8]byte + + //unsigned char mac[crypto_onetimeauth_poly1305_BYTES]; + //const unsigned char *c; + //const unsigned char *stored_mac; + //unsigned long long mlen; + //unsigned char tag; + // + //if (mlen_p != NULL) { + //*mlen_p = 0U; + //} + //if (tag_p != NULL) { + //*tag_p = 0xff; + //} + + //if (inlen < crypto_secretstream_xchacha20poly1305_ABYTES) { + //return -1; + //} + if inlen < StreamABytes { + return nil, 0, invalidInput + } + //mlen = inlen - crypto_secretstream_xchacha20poly1305_ABYTES; + mlen := inlen - StreamABytes + + //if (mlen > crypto_secretstream_xchacha20poly1305_MESSAGEBYTES_MAX) { + //sodium_misuse(); + //} + + chacha, err := chacha20.NewUnauthenticatedCipher(s.k[:], s.nonce[:]) + if err != nil { + return nil, 0, err + } + //crypto_stream_chacha20_ietf(block, sizeof block, state->nonce, state->k); + chacha.XORKeyStream(block[:], block[:]) + + //crypto_onetimeauth_poly1305_init(&poly1305_state, block); + var poly_init [32]byte + copy(poly_init[:], block[:]) + poly := poly1305.New(&poly_init) + + // TODO + //sodium_memzero(block, sizeof block); + //crypto_onetimeauth_poly1305_update(&poly1305_state, ad, adlen); + //crypto_onetimeauth_poly1305_update(&poly1305_state, _pad0, + //(0x10 - adlen) & 0xf); + // + + //memset(block, 0, sizeof block); + memzero(block[:]) + //block[0] = in[0]; + block[0] = in[0] + + //crypto_stream_chacha20_ietf_xor_ic(block, block, sizeof block, state->nonce, 1U, state->k); + chacha.XORKeyStream(block[:], block[:]) + //tag = block[0]; + tag := block[0] + //block[0] = in[0]; + block[0] = in[0] + //crypto_onetimeauth_poly1305_update(&poly1305_state, block, sizeof block); + if _, err = poly.Write(block[:]); err != nil { + return nil, 0, err + } + + // + //c = in + (sizeof tag); + c := in[1:] + //crypto_onetimeauth_poly1305_update(&poly1305_state, c, mlen); + if _, err = poly.Write(c[:mlen]); err != nil { + return nil, 0, err + } + + //crypto_onetimeauth_poly1305_update (&poly1305_state, _pad0, (0x10 - (sizeof block) + mlen) & 0xf); + padlen := (0x10 - len(block) + mlen) & 0xf + if _, err = poly.Write(pad0[:padlen]); err != nil { + return nil, 0, err + } + + // + //STORE64_LE(slen, (uint64_t) adlen); + //crypto_onetimeauth_poly1305_update(&poly1305_state, slen, sizeof slen); + binary.LittleEndian.PutUint64(slen[:], uint64(0)) + if _, err = poly.Write(slen[:]); err != nil { + return nil, 0, err + } + + //STORE64_LE(slen, (sizeof block) + mlen); + //crypto_onetimeauth_poly1305_update(&poly1305_state, slen, sizeof slen); + binary.LittleEndian.PutUint64(slen[:], uint64(len(block)+mlen)) + if _, err = poly.Write(slen[:]); err != nil { + return nil, 0, err + } + + // + //crypto_onetimeauth_poly1305_final(&poly1305_state, mac); + //sodium_memzero(&poly1305_state, sizeof poly1305_state); + mac := poly.Sum(nil) + // + //stored_mac = c + mlen; + stored_mac := c[mlen:] + //if (sodium_memcmp(mac, stored_mac, sizeof mac) != 0) { + //sodium_memzero(mac, sizeof mac); + //return -1; + //} + if !bytes.Equal(mac, stored_mac) { + return nil, 0, cryptoFailure + } + // + //crypto_stream_chacha20_ietf_xor_ic(m, c, mlen, state->nonce, 2U, state->k); + m := make([]byte, mlen) + chacha.XORKeyStream(m, c[:mlen]) + + //XOR_BUF(STATE_INONCE(state), mac, crypto_secretstream_xchacha20poly1305_INONCEBYTES); + //sodium_increment(STATE_COUNTER(state), crypto_secretstream_xchacha20poly1305_COUNTERBYTES); + xor_buf(s.nonce[crypto_secretstream_xchacha20poly1305_COUNTERBYTES:], mac) + buf_inc(s.nonce[:crypto_secretstream_xchacha20poly1305_COUNTERBYTES]) + + // TODO + //if ((tag & crypto_secretstream_xchacha20poly1305_TAG_REKEY) != 0 || + //sodium_is_zero(STATE_COUNTER(state), + //crypto_secretstream_xchacha20poly1305_COUNTERBYTES)) { + //crypto_secretstream_xchacha20poly1305_rekey(state); + //} + + //if (mlen_p != NULL) { + //*mlen_p = mlen; + //} + //if (tag_p != NULL) { + //*tag_p = tag; + //} + //return 0; + return m, tag, nil +} + +func memzero(b []byte) { + for i := range b { + b[i] = 0 + } +} + +func xor_buf(out, in []byte) { + for i := range out { + out[i] ^= in[i] + } +} + +func buf_inc(n []byte) { + c := 1 + + for i := range n { + c += int(n[i]) + n[i] = byte(c) + c >>= 8 + } +} From 6a60e3e1b23bd465c18d4075c0de7816141f5766 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 20 Oct 2023 13:40:23 +0530 Subject: [PATCH 119/183] Add test for secret box --- internal/crypto/crypto_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/internal/crypto/crypto_test.go b/internal/crypto/crypto_test.go index 9f3c60bea..9fa90ec72 100644 --- a/internal/crypto/crypto_test.go +++ b/internal/crypto/crypto_test.go @@ -51,3 +51,20 @@ func TestDecryptChaCha20poly1305(t *testing.T) { t.Fatalf("Decrypted text : %s does not match the expected text: %s", string(decryptedText), expectedPlainText) } } + +// Write a test for SecretBoxOpenBase64 +func TestSecretBoxOpenBase64(t *testing.T) { + sealedCipherText := "KHwRN+RzvTu+jC7mCdkMsqnTPSLvevtZILmcR2OYFbIRPqDyjAl+m8KxD9B5fiEo" + sealNonce := "jgfPDOsQh2VdIHWJVSBicMPF2sQW3HIY" + sealKey, _ := base64.StdEncoding.DecodeString("kercNpvGufMTTHmDwAhz26DgCAvznd1+/buBqKEkWr4=") + expectedSealedText := "O1ObUBMv+SCE1qWHD7+WViEIZcAeTp18Y+m9eMlDE1Y=" + + plainText, err := SecretBoxOpenBase64(sealedCipherText, sealNonce, sealKey) + if err != nil { + t.Fatalf("Failed to decrypt: %v", err) + } + + if expectedSealedText != base64.StdEncoding.EncodeToString(plainText) { + t.Fatalf("Decrypted text : %s does not match the expected text: %s", string(plainText), expectedSealedText) + } +} From 0b7752688439d0afa56ce0aeb80f0844ca80ec77 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 20 Oct 2023 13:46:24 +0530 Subject: [PATCH 120/183] Add test for encrypt and decrypt chacha --- internal/crypto/crypto_test.go | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/internal/crypto/crypto_test.go b/internal/crypto/crypto_test.go index 9fa90ec72..fc48fa768 100644 --- a/internal/crypto/crypto_test.go +++ b/internal/crypto/crypto_test.go @@ -1,6 +1,7 @@ package crypto import ( + "crypto/rand" "encoding/base64" "testing" ) @@ -32,7 +33,6 @@ func TestDecryptChaCha20poly1305(t *testing.T) { if err != nil { t.Fatalf("Failed to derive key: %v", err) } - decodedCipherText, err := base64.StdEncoding.DecodeString(cipherText) if err != nil { t.Fatalf("Failed to decode cipher text: %v", err) @@ -52,7 +52,25 @@ func TestDecryptChaCha20poly1305(t *testing.T) { } } -// Write a test for SecretBoxOpenBase64 +func TestEncryptAndDecryptChaCha20Ploy1305(t *testing.T) { + key := make([]byte, 32) + _, err := rand.Read(key) + if err != nil { + t.Fatalf("Failed to generate random key: %v", err) + } + cipher, nonce, err := EncryptChaCha20poly1305([]byte("plain_text"), key) + if err != nil { + return + } + plainText, err := decryptChaCha20poly1305(cipher, key, nonce) + if err != nil { + t.Fatalf("Failed to decrypt: %v", err) + } + if string(plainText) != "plain_text" { + t.Fatalf("Decrypted text : %s does not match the expected text: %s", string(plainText), "plain_text") + } +} + func TestSecretBoxOpenBase64(t *testing.T) { sealedCipherText := "KHwRN+RzvTu+jC7mCdkMsqnTPSLvevtZILmcR2OYFbIRPqDyjAl+m8KxD9B5fiEo" sealNonce := "jgfPDOsQh2VdIHWJVSBicMPF2sQW3HIY" From f7bae04831690314e03a85063a6abb6e9c4432d7 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 20 Oct 2023 13:54:45 +0530 Subject: [PATCH 121/183] NativeGo: Add method for encryptChacha --- internal/crypto/crypto_libsodium.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/internal/crypto/crypto_libsodium.go b/internal/crypto/crypto_libsodium.go index 04b87844c..9cc8c1f76 100644 --- a/internal/crypto/crypto_libsodium.go +++ b/internal/crypto/crypto_libsodium.go @@ -20,7 +20,7 @@ import ( // - A byte slice representing the encrypted data. // - A byte slice representing the header of the encrypted data. // - An error object, which is nil if no error occurs. -func EncryptChaCha20poly1305(data []byte, key []byte) ([]byte, []byte, error) { +func EncryptChaCha20poly1305LibSodium(data []byte, key []byte) ([]byte, []byte, error) { var buf bytes.Buffer encoder := sodium.MakeSecretStreamXCPEncoder(sodium.SecretStreamXCPKey{Bytes: key}, &buf) _, err := encoder.WriteAndClose(data) @@ -31,6 +31,18 @@ func EncryptChaCha20poly1305(data []byte, key []byte) ([]byte, []byte, error) { return buf.Bytes(), encoder.Header().Bytes, nil } +func EncryptChaCha20poly1305(data []byte, key []byte) ([]byte, []byte, error) { + encryptor, header, err := NewEncryptor(key) + if err != nil { + return nil, nil, err + } + encoded, err := encryptor.Push(data, TagFinal) + if err != nil { + return nil, nil, err + } + return encoded, header, nil +} + // decryptChaCha20poly1305 decrypts the given data using the ChaCha20-Poly1305 algorithm. // Parameters: // - data: The encrypted data as a byte slice. From 8a7278b5e61d674487bf3f1d740bbb8e3b23d44e Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 20 Oct 2023 14:55:20 +0530 Subject: [PATCH 122/183] Commend out decrypt file logic --- internal/crypto/crypto.go | 9 ++ internal/crypto/crypto_libsodium.go | 241 ++++++++++++++++------------ 2 files changed, 150 insertions(+), 100 deletions(-) diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go index b757c996b..83200896b 100644 --- a/internal/crypto/crypto.go +++ b/internal/crypto/crypto.go @@ -3,6 +3,7 @@ package crypto import ( "encoding/base64" "encoding/binary" + "errors" "fmt" "github.com/minio/blake2b-simd" "golang.org/x/crypto/argon2" @@ -20,8 +21,16 @@ const ( cryptoKDFBlake2bBytesMax = 64 cryptoGenerichashBlake2bSaltBytes = 16 cryptoGenerichashBlake2bPersonalBytes = 16 + BoxSealBytes = 48 // 32 for the ephemeral public key + 16 for the MAC ) +var ( + ErrOpenBox = errors.New("failed to open box") + ErrSealedOpenBox = errors.New("failed to open sealed box") +) + +const () + // DeriveArgonKey generates a 32-bit cryptographic key using the Argon2id algorithm. // Parameters: // - password: The plaintext password to be hashed. diff --git a/internal/crypto/crypto_libsodium.go b/internal/crypto/crypto_libsodium.go index 9cc8c1f76..a65f4489a 100644 --- a/internal/crypto/crypto_libsodium.go +++ b/internal/crypto/crypto_libsodium.go @@ -1,16 +1,22 @@ package crypto import ( - "bufio" - "bytes" "cli-go/utils/encoding" - "fmt" - "github.com/jamesruan/sodium" - "io" - "log" - "os" + "golang.org/x/crypto/nacl/box" + "golang.org/x/crypto/nacl/secretbox" ) +//func EncryptChaCha20poly1305LibSodium(data []byte, key []byte) ([]byte, []byte, error) { +// var buf bytes.Buffer +// encoder := sodium.MakeSecretStreamXCPEncoder(sodium.SecretStreamXCPKey{Bytes: key}, &buf) +// _, err := encoder.WriteAndClose(data) +// if err != nil { +// log.Println("Failed to write to encoder", err) +// return nil, nil, err +// } +// return buf.Bytes(), encoder.Header().Bytes, nil +//} + // EncryptChaCha20poly1305 encrypts the given data using the ChaCha20-Poly1305 algorithm. // Parameters: // - data: The plaintext data as a byte slice. @@ -20,17 +26,6 @@ import ( // - A byte slice representing the encrypted data. // - A byte slice representing the header of the encrypted data. // - An error object, which is nil if no error occurs. -func EncryptChaCha20poly1305LibSodium(data []byte, key []byte) ([]byte, []byte, error) { - var buf bytes.Buffer - encoder := sodium.MakeSecretStreamXCPEncoder(sodium.SecretStreamXCPKey{Bytes: key}, &buf) - _, err := encoder.WriteAndClose(data) - if err != nil { - log.Println("Failed to write to encoder", err) - return nil, nil, err - } - return buf.Bytes(), encoder.Header().Bytes, nil -} - func EncryptChaCha20poly1305(data []byte, key []byte) ([]byte, []byte, error) { encryptor, header, err := NewEncryptor(key) if err != nil { @@ -52,26 +47,26 @@ func EncryptChaCha20poly1305(data []byte, key []byte) ([]byte, []byte, error) { // Returns: // - A byte slice representing the decrypted data. // - An error object, which is nil if no error occurs. -func decryptChaCha20poly1305LibSodium(data []byte, key []byte, nonce []byte) ([]byte, error) { - reader := bytes.NewReader(data) - header := sodium.SecretStreamXCPHeader{Bytes: nonce} - decoder, err := sodium.MakeSecretStreamXCPDecoder( - sodium.SecretStreamXCPKey{Bytes: key}, - reader, - header) - if err != nil { - log.Println("Failed to make secret stream decoder", err) - return nil, err - } - // Buffer to store the decrypted data - decryptedData := make([]byte, len(data)) - n, err := decoder.Read(decryptedData) - if err != nil && err != io.EOF { - log.Println("Failed to read from decoder", err) - return nil, err - } - return decryptedData[:n], nil -} +//func decryptChaCha20poly1305LibSodium(data []byte, key []byte, nonce []byte) ([]byte, error) { +// reader := bytes.NewReader(data) +// header := sodium.SecretStreamXCPHeader{Bytes: nonce} +// decoder, err := sodium.MakeSecretStreamXCPDecoder( +// sodium.SecretStreamXCPKey{Bytes: key}, +// reader, +// header) +// if err != nil { +// log.Println("Failed to make secret stream decoder", err) +// return nil, err +// } +// // Buffer to store the decrypted data +// decryptedData := make([]byte, len(data)) +// n, err := decoder.Read(decryptedData) +// if err != nil && err != io.EOF { +// log.Println("Failed to read from decoder", err) +// return nil, err +// } +// return decryptedData[:n], nil +//} func decryptChaCha20poly1305(data []byte, key []byte, nonce []byte) ([]byte, error) { decryptor, err := NewDecryptor(key, nonce) @@ -85,75 +80,121 @@ func decryptChaCha20poly1305(data []byte, key []byte, nonce []byte) ([]byte, err return decoded, nil } -func SecretBoxOpen(c []byte, n []byte, k []byte) ([]byte, error) { - var cp sodium.Bytes = c - res, err := cp.SecretBoxOpen(sodium.SecretBoxNonce{Bytes: n}, sodium.SecretBoxKey{Bytes: k}) - return res, err -} +//func SecretBoxOpenLibSodium(c []byte, n []byte, k []byte) ([]byte, error) { +// var cp sodium.Bytes = c +// res, err := cp.SecretBoxOpen(sodium.SecretBoxNonce{Bytes: n}, sodium.SecretBoxKey{Bytes: k}) +// return res, err +//} func SecretBoxOpenBase64(cipher string, nonce string, k []byte) ([]byte, error) { return SecretBoxOpen(encoding.DecodeBase64(cipher), encoding.DecodeBase64(nonce), k) } -func SealedBoxOpen(cipherText []byte, publicKey, masterSecret []byte) ([]byte, error) { - var cp sodium.Bytes = cipherText - om, err := cp.SealedBoxOpen(sodium.BoxKP{ - PublicKey: sodium.BoxPublicKey{Bytes: publicKey}, - SecretKey: sodium.BoxSecretKey{Bytes: masterSecret}, - }) - if err != nil { - return nil, fmt.Errorf("failed to open sealed box: %v", err) +func SecretBoxOpen(c []byte, n []byte, k []byte) ([]byte, error) { + // Check for valid lengths of nonce and key + if len(n) != 24 || len(k) != 32 { + return nil, ErrOpenBox } - return om, nil + + var nonce [24]byte + var key [32]byte + copy(nonce[:], n) + copy(key[:], k) + + // Decrypt the message using Go's nacl/secretbox + decrypted, ok := secretbox.Open(nil, c, &nonce, &key) + if !ok { + return nil, ErrOpenBox + } + + return decrypted, nil +} + +//func SealedBoxOpenLib(cipherText []byte, publicKey, masterSecret []byte) ([]byte, error) { +// var cp sodium.Bytes = cipherText +// om, err := cp.SealedBoxOpen(sodium.BoxKP{ +// PublicKey: sodium.BoxPublicKey{Bytes: publicKey}, +// SecretKey: sodium.BoxSecretKey{Bytes: masterSecret}, +// }) +// if err != nil { +// return nil, fmt.Errorf("failed to open sealed box: %v", err) +// } +// return om, nil +//} + +func SealedBoxOpen(cipherText, publicKey, masterSecret []byte) ([]byte, error) { + if len(cipherText) < BoxSealBytes { + return nil, ErrOpenBox + } + + // Extract ephemeral public key from the ciphertext + var ephemeralPublicKey [32]byte + copy(ephemeralPublicKey[:], publicKey[:32]) + + // Extract ephemeral public key from the ciphertext + var masterKey [32]byte + copy(masterKey[:], masterSecret[:32]) + + // Decrypt the message using nacl/box + decrypted, ok := box.OpenAnonymous(nil, cipherText, &ephemeralPublicKey, &masterKey) + if !ok { + return nil, ErrOpenBox + } + + return decrypted, nil } func DecryptFile(encryptedFilePath string, decryptedFilePath string, key, nonce []byte) error { - inputFile, err := os.Open(encryptedFilePath) - if err != nil { - return err - } - defer inputFile.Close() - - outputFile, err := os.Create(decryptedFilePath) - if err != nil { - return err - } - defer outputFile.Close() - - reader := bufio.NewReader(inputFile) - writer := bufio.NewWriter(outputFile) - - header := sodium.SecretStreamXCPHeader{Bytes: nonce} - decoder, err := sodium.MakeSecretStreamXCPDecoder( - sodium.SecretStreamXCPKey{Bytes: key}, - reader, - header) - if err != nil { - log.Println("Failed to make secret stream decoder", err) - return err - } - - buf := make([]byte, decryptionBufferSize) - for { - n, errErr := decoder.Read(buf) - if errErr != nil && errErr != io.EOF { - log.Println("Failed to read from decoder", errErr) - return errErr - } - if n == 0 { - break - } - if _, err := writer.Write(buf[:n]); err != nil { - log.Println("Failed to write to output file", err) - return err - } - if errErr == io.EOF { - break - } - } - if err := writer.Flush(); err != nil { - log.Println("Failed to flush writer", err) - return err - } - return nil + panic("not implemented decrypt file") } + +//func DecryptFileLib(encryptedFilePath string, decryptedFilePath string, key, nonce []byte) error { +// inputFile, err := os.Open(encryptedFilePath) +// if err != nil { +// return err +// } +// defer inputFile.Close() +// +// outputFile, err := os.Create(decryptedFilePath) +// if err != nil { +// return err +// } +// defer outputFile.Close() +// +// reader := bufio.NewReader(inputFile) +// writer := bufio.NewWriter(outputFile) +// +// header := sodium.SecretStreamXCPHeader{Bytes: nonce} +// decoder, err := sodium.MakeSecretStreamXCPDecoder( +// sodium.SecretStreamXCPKey{Bytes: key}, +// reader, +// header) +// if err != nil { +// log.Println("Failed to make secret stream decoder", err) +// return err +// } +// +// buf := make([]byte, decryptionBufferSize) +// for { +// n, errErr := decoder.Read(buf) +// if errErr != nil && errErr != io.EOF { +// log.Println("Failed to read from decoder", errErr) +// return errErr +// } +// if n == 0 { +// break +// } +// if _, err := writer.Write(buf[:n]); err != nil { +// log.Println("Failed to write to output file", err) +// return err +// } +// if errErr == io.EOF { +// break +// } +// } +// if err := writer.Flush(); err != nil { +// log.Println("Failed to flush writer", err) +// return err +// } +// return nil +//} From 2ffeda16c80fbc069a4de714fada52629df68d99 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 20 Oct 2023 14:56:13 +0530 Subject: [PATCH 123/183] Release: Build binary for all standard os+arch --- release.sh | 46 ++++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/release.sh b/release.sh index fb74e3731..7c5047385 100755 --- a/release.sh +++ b/release.sh @@ -6,40 +6,38 @@ mkdir -p bin # List of target operating systems OS_TARGETS=("windows" "linux" "darwin") +# Corresponding architectures for each OS +ARCH_TARGETS=("386 amd64" "386 amd64 arm arm64" "amd64 arm64") + # Loop through each OS target -for OS in "${OS_TARGETS[@]}" +for index in "${!OS_TARGETS[@]}" do - # Set the GOOS environment variable for the current target OS - export GOOS="$OS" + OS=${OS_TARGETS[$index]} + for ARCH in ${ARCH_TARGETS[$index]} + do + # Set the GOOS environment variable for the current target OS + export GOOS="$OS" + export GOARCH="$ARCH" - # Set the output binary name to "ente-cli" for the current OS - BINARY_NAME="ente-cli" - - # Add .exe extension for Windows - if [ "$OS" == "windows" ]; then - BINARY_NAME="ente-cli.exe" - fi + # Set the output binary name to "ente-cli" for the current OS and architecture + BINARY_NAME="ente-cli-$OS-$ARCH" # Add .exe extension for Windows - if [ "$OS" == "darwin" ]; then - BINARY_NAME="ente-cli-mac" - fi + if [ "$OS" == "windows" ]; then + BINARY_NAME="ente-cli-$OS-$ARCH.exe" + fi - if [ "$OS" == "linux" ]; then - BINARY_NAME="ente-cli-linux" - fi - # make bin directory if it doesn't exist - mkdir -p bin + # Build the binary and place it in the "bin" directory + go build -o "bin/$BINARY_NAME" main.go - # Build the binary and place it in the "bin" directory - go build -o "bin/$BINARY_NAME" main.go - - # Print a message indicating the build is complete for the current OS - echo "Built for $OS as bin/$BINARY_NAME" + # Print a message indicating the build is complete for the current OS and architecture + echo "Built for $OS ($ARCH) as bin/$BINARY_NAME" + done done # Clean up any environment variables unset GOOS +unset GOARCH # Print a message indicating the build process is complete -echo "Build process completed for all platforms. Binaries are in the 'bin' directory." +echo "Build process completed for all platforms and architectures. Binaries are in the 'bin' directory." From f05be385d554fb896a085dfaadaaf58e6eda0a29 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 21 Oct 2023 08:41:56 +0530 Subject: [PATCH 124/183] Minor refactor --- internal/crypto/stream.go | 41 ++++++++++++++++++++------------------- internal/crypto/utils.go | 7 +++++++ 2 files changed, 28 insertions(+), 20 deletions(-) create mode 100644 internal/crypto/utils.go diff --git a/internal/crypto/stream.go b/internal/crypto/stream.go index 6d4359b16..13ecef372 100644 --- a/internal/crypto/stream.go +++ b/internal/crypto/stream.go @@ -13,10 +13,15 @@ import ( // public constants const ( + //TagMessage the most common tag, that doesn't add any information about the nature of the message. TagMessage = 0 - TagPush = 0x01 - TagRekey = 0x02 - TagFinal = TagPush | TagRekey + // TagPush indicates that the message marks the end of a set of messages, + // but not the end of the stream. For example, a huge JSON string sent as multiple chunks can use this tag to indicate to the application that the string is complete and that it can be decoded. But the stream itself is not closed, and more data may follow. + TagPush = 0x01 + // TagRekey "forget" the key used to encrypt this message and the previous ones, and derive a new secret key. + TagRekey = 0x02 + // TagFinal indicates that the message marks the end of the stream, and erases the secret key used to encrypt the previous sequence. + TagFinal = TagPush | TagRekey StreamKeyBytes = chacha20poly1305.KeySize StreamHeaderBytes = chacha20poly1305.NonceSizeX @@ -34,6 +39,7 @@ var invalidKey = errors.New("invalid key") var invalidInput = errors.New("invalid input") var cryptoFailure = errors.New("crypto failed") +// crypto_secretstream_xchacha20poly1305_state type streamState struct { k [StreamKeyBytes]byte nonce [chacha20poly1305.NonceSize]byte @@ -145,7 +151,7 @@ func (s *encryptor) Push(plain []byte, tag byte) ([]byte, error) { //memset(block, 0, sizeof block); //block[0] = tag; - memzero(block[:]) + memZero(block[:]) block[0] = tag // @@ -233,13 +239,14 @@ func NewDecryptor(key, header []byte) (Decryptor, error) { return stream, nil } -func (s *decryptor) Pull(in []byte) ([]byte, byte, error) { - inlen := len(in) +func (s *decryptor) Pull(cipher []byte) ([]byte, byte, error) { + inlen := len(cipher) + //crypto_onetimeauth_poly1305_state poly1305_state; + var poly1305State [32]byte //unsigned char block[64U]; var block [64]byte - //unsigned char slen[8U]; var slen [8]byte @@ -277,9 +284,9 @@ func (s *decryptor) Pull(in []byte) ([]byte, byte, error) { chacha.XORKeyStream(block[:], block[:]) //crypto_onetimeauth_poly1305_init(&poly1305_state, block); - var poly_init [32]byte - copy(poly_init[:], block[:]) - poly := poly1305.New(&poly_init) + + copy(poly1305State[:], block[:]) + poly := poly1305.New(&poly1305State) // TODO //sodium_memzero(block, sizeof block); @@ -289,16 +296,16 @@ func (s *decryptor) Pull(in []byte) ([]byte, byte, error) { // //memset(block, 0, sizeof block); - memzero(block[:]) + memZero(block[:]) //block[0] = in[0]; - block[0] = in[0] + block[0] = cipher[0] //crypto_stream_chacha20_ietf_xor_ic(block, block, sizeof block, state->nonce, 1U, state->k); chacha.XORKeyStream(block[:], block[:]) //tag = block[0]; tag := block[0] //block[0] = in[0]; - block[0] = in[0] + block[0] = cipher[0] //crypto_onetimeauth_poly1305_update(&poly1305_state, block, sizeof block); if _, err = poly.Write(block[:]); err != nil { return nil, 0, err @@ -306,7 +313,7 @@ func (s *decryptor) Pull(in []byte) ([]byte, byte, error) { // //c = in + (sizeof tag); - c := in[1:] + c := cipher[1:] //crypto_onetimeauth_poly1305_update(&poly1305_state, c, mlen); if _, err = poly.Write(c[:mlen]); err != nil { return nil, 0, err @@ -374,12 +381,6 @@ func (s *decryptor) Pull(in []byte) ([]byte, byte, error) { return m, tag, nil } -func memzero(b []byte) { - for i := range b { - b[i] = 0 - } -} - func xor_buf(out, in []byte) { for i := range out { out[i] ^= in[i] diff --git a/internal/crypto/utils.go b/internal/crypto/utils.go new file mode 100644 index 000000000..9491e99c8 --- /dev/null +++ b/internal/crypto/utils.go @@ -0,0 +1,7 @@ +package crypto + +func memZero(b []byte) { + for i := range b { + b[i] = 0 + } +} From 33bda4cbb7ab748836955c257548f0db9d7f98e8 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 21 Oct 2023 09:00:33 +0530 Subject: [PATCH 125/183] Add tag validation during decrypt --- internal/crypto/stream.go | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/internal/crypto/stream.go b/internal/crypto/stream.go index 13ecef372..7b1d92ffe 100644 --- a/internal/crypto/stream.go +++ b/internal/crypto/stream.go @@ -25,7 +25,8 @@ const ( StreamKeyBytes = chacha20poly1305.KeySize StreamHeaderBytes = chacha20poly1305.NonceSizeX - StreamABytes = 16 + 1 + // crypto_secretstream_xchacha20poly1305_ABYTES + XChaCha20Poly1305IetfABYTES = 16 + 1 ) const crypto_core_hchacha20_INPUTBYTES = 16 @@ -129,7 +130,7 @@ func (s *encryptor) Push(plain []byte, tag byte) ([]byte, error) { //sodium_misuse(); //} - out := make([]byte, mlen+StreamABytes) + out := make([]byte, mlen+XChaCha20Poly1305IetfABYTES) chacha, err := chacha20.NewUnauthenticatedCipher(s.k[:], s.nonce[:]) if err != nil { @@ -240,7 +241,7 @@ func NewDecryptor(key, header []byte) (Decryptor, error) { } func (s *decryptor) Pull(cipher []byte) ([]byte, byte, error) { - inlen := len(cipher) + ciperLen := len(cipher) //crypto_onetimeauth_poly1305_state poly1305_state; var poly1305State [32]byte @@ -253,8 +254,8 @@ func (s *decryptor) Pull(cipher []byte) ([]byte, byte, error) { //unsigned char mac[crypto_onetimeauth_poly1305_BYTES]; //const unsigned char *c; //const unsigned char *stored_mac; - //unsigned long long mlen; - //unsigned char tag; + //unsigned long long mlen; // length of the returned message + //unsigned char tag; // for the return value // //if (mlen_p != NULL) { //*mlen_p = 0U; @@ -263,24 +264,26 @@ func (s *decryptor) Pull(cipher []byte) ([]byte, byte, error) { //*tag_p = 0xff; //} - //if (inlen < crypto_secretstream_xchacha20poly1305_ABYTES) { - //return -1; - //} - if inlen < StreamABytes { + /* + if (inlen < crypto_secretstream_xchacha20poly1305_ABYTES) { + return -1; + } + mlen = inlen - crypto_secretstream_xchacha20poly1305_ABYTES; + */ + if ciperLen < XChaCha20Poly1305IetfABYTES { return nil, 0, invalidInput } - //mlen = inlen - crypto_secretstream_xchacha20poly1305_ABYTES; - mlen := inlen - StreamABytes + mlen := ciperLen - XChaCha20Poly1305IetfABYTES //if (mlen > crypto_secretstream_xchacha20poly1305_MESSAGEBYTES_MAX) { //sodium_misuse(); //} + //crypto_stream_chacha20_ietf(block, sizeof block, state->nonce, state->k); chacha, err := chacha20.NewUnauthenticatedCipher(s.k[:], s.nonce[:]) if err != nil { return nil, 0, err } - //crypto_stream_chacha20_ietf(block, sizeof block, state->nonce, state->k); chacha.XORKeyStream(block[:], block[:]) //crypto_onetimeauth_poly1305_init(&poly1305_state, block); From fe646ca74b5654d8a3c64cc30e54a81601bc1093 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 21 Oct 2023 10:04:20 +0530 Subject: [PATCH 126/183] secretStream: minor refactor --- internal/crypto/crypto_libsodium.go | 6 ++++- internal/crypto/stream.go | 39 +++++++++++++++-------------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/internal/crypto/crypto_libsodium.go b/internal/crypto/crypto_libsodium.go index a65f4489a..021db9ed1 100644 --- a/internal/crypto/crypto_libsodium.go +++ b/internal/crypto/crypto_libsodium.go @@ -2,6 +2,7 @@ package crypto import ( "cli-go/utils/encoding" + "errors" "golang.org/x/crypto/nacl/box" "golang.org/x/crypto/nacl/secretbox" ) @@ -73,7 +74,10 @@ func decryptChaCha20poly1305(data []byte, key []byte, nonce []byte) ([]byte, err if err != nil { return nil, err } - decoded, _, err := decryptor.Pull(data) + decoded, tag, err := decryptor.Pull(data) + if tag != TagFinal { + return nil, errors.New("invalid tag") + } if err != nil { return nil, err } diff --git a/internal/crypto/stream.go b/internal/crypto/stream.go index 7b1d92ffe..c428aa229 100644 --- a/internal/crypto/stream.go +++ b/internal/crypto/stream.go @@ -241,7 +241,7 @@ func NewDecryptor(key, header []byte) (Decryptor, error) { } func (s *decryptor) Pull(cipher []byte) ([]byte, byte, error) { - ciperLen := len(cipher) + cipherLen := len(cipher) //crypto_onetimeauth_poly1305_state poly1305_state; var poly1305State [32]byte @@ -270,10 +270,10 @@ func (s *decryptor) Pull(cipher []byte) ([]byte, byte, error) { } mlen = inlen - crypto_secretstream_xchacha20poly1305_ABYTES; */ - if ciperLen < XChaCha20Poly1305IetfABYTES { + if cipherLen < XChaCha20Poly1305IetfABYTES { return nil, 0, invalidInput } - mlen := ciperLen - XChaCha20Poly1305IetfABYTES + mlen := cipherLen - XChaCha20Poly1305IetfABYTES //if (mlen > crypto_secretstream_xchacha20poly1305_MESSAGEBYTES_MAX) { //sodium_misuse(); @@ -299,30 +299,28 @@ func (s *decryptor) Pull(cipher []byte) ([]byte, byte, error) { // //memset(block, 0, sizeof block); - memZero(block[:]) //block[0] = in[0]; - block[0] = cipher[0] - //crypto_stream_chacha20_ietf_xor_ic(block, block, sizeof block, state->nonce, 1U, state->k); - chacha.XORKeyStream(block[:], block[:]) - //tag = block[0]; - tag := block[0] - //block[0] = in[0]; + memZero(block[:]) block[0] = cipher[0] + chacha.XORKeyStream(block[:], block[:]) + + //tag = block[0]; + //block[0] = in[0]; //crypto_onetimeauth_poly1305_update(&poly1305_state, block, sizeof block); + tag := block[0] + block[0] = cipher[0] if _, err = poly.Write(block[:]); err != nil { return nil, 0, err } - // //c = in + (sizeof tag); - c := cipher[1:] //crypto_onetimeauth_poly1305_update(&poly1305_state, c, mlen); + //crypto_onetimeauth_poly1305_update (&poly1305_state, _pad0, (0x10 - (sizeof block) + mlen) & 0xf); + c := cipher[1:] if _, err = poly.Write(c[:mlen]); err != nil { return nil, 0, err } - - //crypto_onetimeauth_poly1305_update (&poly1305_state, _pad0, (0x10 - (sizeof block) + mlen) & 0xf); padlen := (0x10 - len(block) + mlen) & 0xf if _, err = poly.Write(pad0[:padlen]); err != nil { return nil, 0, err @@ -346,24 +344,27 @@ func (s *decryptor) Pull(cipher []byte) ([]byte, byte, error) { // //crypto_onetimeauth_poly1305_final(&poly1305_state, mac); //sodium_memzero(&poly1305_state, sizeof poly1305_state); + mac := poly.Sum(nil) - // + memZero(poly1305State[:]) + //stored_mac = c + mlen; - stored_mac := c[mlen:] //if (sodium_memcmp(mac, stored_mac, sizeof mac) != 0) { //sodium_memzero(mac, sizeof mac); //return -1; //} + stored_mac := c[mlen:] if !bytes.Equal(mac, stored_mac) { + memZero(mac) return nil, 0, cryptoFailure } - // + //crypto_stream_chacha20_ietf_xor_ic(m, c, mlen, state->nonce, 2U, state->k); + //XOR_BUF(STATE_INONCE(state), mac, crypto_secretstream_xchacha20poly1305_INONCEBYTES); + //sodium_increment(STATE_COUNTER(state), crypto_secretstream_xchacha20poly1305_COUNTERBYTES); m := make([]byte, mlen) chacha.XORKeyStream(m, c[:mlen]) - //XOR_BUF(STATE_INONCE(state), mac, crypto_secretstream_xchacha20poly1305_INONCEBYTES); - //sodium_increment(STATE_COUNTER(state), crypto_secretstream_xchacha20poly1305_COUNTERBYTES); xor_buf(s.nonce[crypto_secretstream_xchacha20poly1305_COUNTERBYTES:], mac) buf_inc(s.nonce[:crypto_secretstream_xchacha20poly1305_COUNTERBYTES]) From dad48f0d3a4a174cb83dda3f6177180eef965bda Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 21 Oct 2023 10:12:29 +0530 Subject: [PATCH 127/183] Refactor crypto/secretSteam --- internal/crypto/stream.go | 46 +++++++++++++-------------------------- internal/crypto/utils.go | 16 ++++++++++++++ 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/internal/crypto/stream.go b/internal/crypto/stream.go index c428aa229..5ff47245d 100644 --- a/internal/crypto/stream.go +++ b/internal/crypto/stream.go @@ -25,14 +25,14 @@ const ( StreamKeyBytes = chacha20poly1305.KeySize StreamHeaderBytes = chacha20poly1305.NonceSizeX - // crypto_secretstream_xchacha20poly1305_ABYTES + // XChaCha20Poly1305IetfABYTES links to crypto_secretstream_xchacha20poly1305_ABYTES XChaCha20Poly1305IetfABYTES = 16 + 1 ) -const crypto_core_hchacha20_INPUTBYTES = 16 +const cryptoCoreHchacha20InputBytes = 16 /* const crypto_secretstream_xchacha20poly1305_INONCEBYTES = 8 */ -const crypto_secretstream_xchacha20poly1305_COUNTERBYTES = 4 +const cryptoSecretStreamXchacha20poly1305Counterbytes = 4 var pad0 [16]byte @@ -98,8 +98,8 @@ func NewEncryptor(key []byte) (Encryptor, []byte, error) { stream.pad[i] = 0 } - for i, b := range header[crypto_core_hchacha20_INPUTBYTES:] { - stream.nonce[i+crypto_secretstream_xchacha20poly1305_COUNTERBYTES] = b + for i, b := range header[cryptoCoreHchacha20InputBytes:] { + stream.nonce[i+cryptoSecretStreamXchacha20poly1305Counterbytes] = b } // fmt.Printf("stream: %+v\n", stream.streamState) @@ -195,8 +195,8 @@ func (s *encryptor) Push(plain []byte, tag byte) ([]byte, error) { //XOR_BUF(STATE_INONCE(state), mac, crypto_secretstream_xchacha20poly1305_INONCEBYTES); //sodium_increment(STATE_COUNTER(state), crypto_secretstream_xchacha20poly1305_COUNTERBYTES); - xor_buf(s.nonce[crypto_secretstream_xchacha20poly1305_COUNTERBYTES:], mac) - buf_inc(s.nonce[:crypto_secretstream_xchacha20poly1305_COUNTERBYTES]) + xorBuf(s.nonce[cryptoSecretStreamXchacha20poly1305Counterbytes:], mac) + bufInc(s.nonce[:cryptoSecretStreamXchacha20poly1305Counterbytes]) // TODO //if ((tag & crypto_secretstream_xchacha20poly1305_TAG_REKEY) != 0 || @@ -229,8 +229,8 @@ func NewDecryptor(key, header []byte) (Decryptor, error) { //memcpy(STATE_INONCE(state), in + crypto_core_hchacha20_INPUTBYTES, // crypto_secretstream_xchacha20poly1305_INONCEBYTES); - copy(stream.nonce[crypto_secretstream_xchacha20poly1305_COUNTERBYTES:], - header[crypto_core_hchacha20_INPUTBYTES:]) + copy(stream.nonce[cryptoSecretStreamXchacha20poly1305Counterbytes:], + header[cryptoCoreHchacha20InputBytes:]) //memset(state->_pad, 0, sizeof state->_pad); copy(stream.pad[:], pad0[:]) @@ -321,8 +321,8 @@ func (s *decryptor) Pull(cipher []byte) ([]byte, byte, error) { if _, err = poly.Write(c[:mlen]); err != nil { return nil, 0, err } - padlen := (0x10 - len(block) + mlen) & 0xf - if _, err = poly.Write(pad0[:padlen]); err != nil { + padLen := (0x10 - len(block) + mlen) & 0xf + if _, err = poly.Write(pad0[:padLen]); err != nil { return nil, 0, err } @@ -353,8 +353,8 @@ func (s *decryptor) Pull(cipher []byte) ([]byte, byte, error) { //sodium_memzero(mac, sizeof mac); //return -1; //} - stored_mac := c[mlen:] - if !bytes.Equal(mac, stored_mac) { + storedMac := c[mlen:] + if !bytes.Equal(mac, storedMac) { memZero(mac) return nil, 0, cryptoFailure } @@ -365,8 +365,8 @@ func (s *decryptor) Pull(cipher []byte) ([]byte, byte, error) { m := make([]byte, mlen) chacha.XORKeyStream(m, c[:mlen]) - xor_buf(s.nonce[crypto_secretstream_xchacha20poly1305_COUNTERBYTES:], mac) - buf_inc(s.nonce[:crypto_secretstream_xchacha20poly1305_COUNTERBYTES]) + xorBuf(s.nonce[cryptoSecretStreamXchacha20poly1305Counterbytes:], mac) + bufInc(s.nonce[:cryptoSecretStreamXchacha20poly1305Counterbytes]) // TODO //if ((tag & crypto_secretstream_xchacha20poly1305_TAG_REKEY) != 0 || @@ -384,19 +384,3 @@ func (s *decryptor) Pull(cipher []byte) ([]byte, byte, error) { //return 0; return m, tag, nil } - -func xor_buf(out, in []byte) { - for i := range out { - out[i] ^= in[i] - } -} - -func buf_inc(n []byte) { - c := 1 - - for i := range n { - c += int(n[i]) - n[i] = byte(c) - c >>= 8 - } -} diff --git a/internal/crypto/utils.go b/internal/crypto/utils.go index 9491e99c8..c794eaff8 100644 --- a/internal/crypto/utils.go +++ b/internal/crypto/utils.go @@ -5,3 +5,19 @@ func memZero(b []byte) { b[i] = 0 } } + +func xorBuf(out, in []byte) { + for i := range out { + out[i] ^= in[i] + } +} + +func bufInc(n []byte) { + c := 1 + + for i := range n { + c += int(n[i]) + n[i] = byte(c) + c >>= 8 + } +} From 81daf19175cbc89b91d21a24ac899224d418e4c3 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 21 Oct 2023 10:15:38 +0530 Subject: [PATCH 128/183] Handle db open timeout err --- main.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 624dd45c8..c46572e37 100644 --- a/main.go +++ b/main.go @@ -9,6 +9,7 @@ import ( "cli-go/utils/constants" "fmt" "log" + "strings" ) func main() { @@ -22,7 +23,11 @@ func main() { } db, err := pkg.GetDB(fmt.Sprintf("%sente-cli.db", cliDBPath)) if err != nil { - panic(err) + if strings.Contains(err.Error(), "timeout") { + log.Fatalf("Please close all other instances of the cli and try again\n%v\n", err) + } else { + panic(err) + } } ctrl := pkg.ClICtrl{ Client: api.NewClient(api.Params{ From b0152c842f8613a5e8b5ad80db1572b96a4165ea Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 21 Oct 2023 11:07:55 +0530 Subject: [PATCH 129/183] Add support for decrypting file --- internal/crypto/crypto_libsodium.go | 57 ++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/internal/crypto/crypto_libsodium.go b/internal/crypto/crypto_libsodium.go index 021db9ed1..662e9ffcc 100644 --- a/internal/crypto/crypto_libsodium.go +++ b/internal/crypto/crypto_libsodium.go @@ -1,10 +1,14 @@ package crypto import ( + "bufio" "cli-go/utils/encoding" "errors" "golang.org/x/crypto/nacl/box" "golang.org/x/crypto/nacl/secretbox" + "io" + "log" + "os" ) //func EncryptChaCha20poly1305LibSodium(data []byte, key []byte) ([]byte, []byte, error) { @@ -149,7 +153,58 @@ func SealedBoxOpen(cipherText, publicKey, masterSecret []byte) ([]byte, error) { } func DecryptFile(encryptedFilePath string, decryptedFilePath string, key, nonce []byte) error { - panic("not implemented decrypt file") + inputFile, err := os.Open(encryptedFilePath) + if err != nil { + return err + } + defer inputFile.Close() + + outputFile, err := os.Create(decryptedFilePath) + if err != nil { + return err + } + defer outputFile.Close() + + reader := bufio.NewReader(inputFile) + writer := bufio.NewWriter(outputFile) + + decryptor, err := NewDecryptor(key, nonce) + if err != nil { + return err + } + + buf := make([]byte, decryptionBufferSize) + for { + readCount, err := reader.Read(buf) + if err != nil && err != io.EOF { + log.Println("Failed to read from input file", err) + return err + } + if readCount == 0 { + break + } + n, tag, errErr := decryptor.Pull(buf[:readCount]) + if errErr != nil && errErr != io.EOF { + log.Println("Failed to read from decoder", errErr) + return errErr + } + + if _, err := writer.Write(n); err != nil { + log.Println("Failed to write to output file", err) + return err + } + if errErr == io.EOF { + break + } + if tag == TagFinal { + break + } + } + if err := writer.Flush(); err != nil { + log.Println("Failed to flush writer", err) + return err + } + return nil } //func DecryptFileLib(encryptedFilePath string, decryptedFilePath string, key, nonce []byte) error { From 326522453f1f60fe1cc5da75b01525438cfd28f7 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 21 Oct 2023 12:22:45 +0530 Subject: [PATCH 130/183] Use fixed temp folder --- pkg/cli.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pkg/cli.go b/pkg/cli.go index 5bcc8edc3..5096df3e1 100644 --- a/pkg/cli.go +++ b/pkg/cli.go @@ -5,7 +5,9 @@ import ( "cli-go/pkg/secrets" "fmt" bolt "go.etcd.io/bbolt" + "log" "os" + "path/filepath" ) type ClICtrl struct { @@ -16,11 +18,16 @@ type ClICtrl struct { } func (c *ClICtrl) Init() error { - dir, err := os.MkdirTemp("", "ente-cli-download") - if err != nil { - return err + tempPath := filepath.Join(os.TempDir(), "ente-cli-download") + // create temp folder if not exists + if _, err := os.Stat(tempPath); os.IsNotExist(err) { + err = os.Mkdir(tempPath, 0755) + if err != nil { + return err + } } - c.tempFolder = dir + log.Printf("Using temp folder %s", tempPath) + c.tempFolder = tempPath return c.DB.Update(func(tx *bolt.Tx) error { _, err := tx.CreateBucketIfNotExists([]byte(AccBucket)) if err != nil { From cc2274b170e436492ee5fd24c2a22542fc716f62 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 21 Oct 2023 12:32:01 +0530 Subject: [PATCH 131/183] Download: Check if file already exists in temp directory --- pkg/download.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pkg/download.go b/pkg/download.go index 70a98a018..a5231ba3b 100644 --- a/pkg/download.go +++ b/pkg/download.go @@ -22,13 +22,19 @@ func (c *ClICtrl) downloadAndDecrypt( ) (*string, error) { dir := c.tempFolder downloadPath := fmt.Sprintf("%s/%d", dir, file.ID) - log.Printf("Downloading %s (%s)", file.GetTitle(), utils.ByteCountDecimal(file.Info.FileSize)) - err := c.Client.DownloadFile(ctx, file.ID, downloadPath) - if err != nil { - return nil, fmt.Errorf("error downloading file %d: %w", file.ID, err) + // check if file exists + if _, err := os.Stat(downloadPath); err == nil { + log.Printf("File already exists %s (%s)", file.GetTitle(), utils.ByteCountDecimal(file.Info.FileSize)) + + } else { + log.Printf("Downloading %s (%s)", file.GetTitle(), utils.ByteCountDecimal(file.Info.FileSize)) + err := c.Client.DownloadFile(ctx, file.ID, downloadPath) + if err != nil { + return nil, fmt.Errorf("error downloading file %d: %w", file.ID, err) + } } decryptedPath := fmt.Sprintf("%s/%d.decrypted", dir, file.ID) - err = crypto.DecryptFile(downloadPath, decryptedPath, file.Key.MustDecrypt(deviceKey), encoding.DecodeBase64(file.FileNonce)) + err := crypto.DecryptFile(downloadPath, decryptedPath, file.Key.MustDecrypt(deviceKey), encoding.DecodeBase64(file.FileNonce)) if err != nil { log.Printf("Error decrypting file %d: %s", file.ID, err) return nil, model.ErrDecryption From 21792a6846389bf13c50d13da99e393e0ae6c60b Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 21 Oct 2023 12:45:13 +0530 Subject: [PATCH 132/183] Fix downloaded cache check --- pkg/download.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/download.go b/pkg/download.go index a5231ba3b..3bd39cb04 100644 --- a/pkg/download.go +++ b/pkg/download.go @@ -23,9 +23,8 @@ func (c *ClICtrl) downloadAndDecrypt( dir := c.tempFolder downloadPath := fmt.Sprintf("%s/%d", dir, file.ID) // check if file exists - if _, err := os.Stat(downloadPath); err == nil { + if stat, err := os.Stat(downloadPath); err == nil && stat.Size() == file.Info.FileSize { log.Printf("File already exists %s (%s)", file.GetTitle(), utils.ByteCountDecimal(file.Info.FileSize)) - } else { log.Printf("Downloading %s (%s)", file.GetTitle(), utils.ByteCountDecimal(file.Info.FileSize)) err := c.Client.DownloadFile(ctx, file.ID, downloadPath) From af34bb9453716346c36ffcbe1b5f71e51e5b2e48 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 21 Oct 2023 12:55:30 +0530 Subject: [PATCH 133/183] Fixed bug in downloading file --- internal/crypto/crypto_libsodium.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/crypto/crypto_libsodium.go b/internal/crypto/crypto_libsodium.go index 662e9ffcc..2c777b2c4 100644 --- a/internal/crypto/crypto_libsodium.go +++ b/internal/crypto/crypto_libsodium.go @@ -173,7 +173,7 @@ func DecryptFile(encryptedFilePath string, decryptedFilePath string, key, nonce return err } - buf := make([]byte, decryptionBufferSize) + buf := make([]byte, decryptionBufferSize+XChaCha20Poly1305IetfABYTES) for { readCount, err := reader.Read(buf) if err != nil && err != io.EOF { From 239ffac930566c540fe8e904d183c828300c193e Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 21 Oct 2023 14:26:32 +0530 Subject: [PATCH 134/183] Update readme --- README.md | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 5371a1790..e005a8dd9 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,24 @@ -# cli tool for exporting data from ente.io +# cli for exporting ente.io data -#### You can configure multiple accounts for export +## Install + +You can either download the binary from the [release page](https://github.com/ente-io/cli/releases) or build it yourself. + +### Build from source + +```shell + go build -o "bin/ente-cli" main.go +``` ### Getting Started -#### Accounts +Run the help command to see all available commands. +```shell +ente-cli --help +``` +#### Accounts +If you wish, you can add multiple accounts (your own and your family members) and export all using this tool. * Add an account ```shell ente-cli account add @@ -21,7 +34,7 @@ ente-cli account update --email yourEmail@example.com --dir ~/photos ``` -## Export +### Export * Start export ```shell ente-cli export @@ -53,7 +66,7 @@ exec into the container ``` -## Build locally +## Releases Run the release script to build the binary and run it. @@ -61,12 +74,3 @@ Run the release script to build the binary and run it. ./release.sh ``` -or you can run the following command - -```shell - go build -o "bin/ente-cli" main.go -``` - -```shell -./bin/ente-cli --help -``` From 300150f433c903e7176098712637ab8852f39102 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 21 Oct 2023 14:28:19 +0530 Subject: [PATCH 135/183] Init goreleaser --- .gitignore | 3 ++- .goreleaser.yaml | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 .goreleaser.yaml diff --git a/.gitignore b/.gitignore index c90d2b2b9..f156a51d7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ scratch/** main config.yaml ente-cli.db -bin/** \ No newline at end of file +bin/** +dist/ diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 000000000..d3fb70f3b --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,44 @@ +# This is an example .goreleaser.yml file with some sensible defaults. +# Make sure to check the documentation at https://goreleaser.com + +# The lines bellow are called `modelines`. See `:help modeline` +# Feel free to remove those if you don't want/need to use them. +# yaml-language-server: $schema=https://goreleaser.com/static/schema.json +# vim: set ts=2 sw=2 tw=0 fo=cnqoj + +before: + hooks: + # You may remove this if you don't use go modules. + - go mod tidy + # you may remove this if you don't need go generate + - go generate ./... + +builds: + - env: + - CGO_ENABLED=0 + goos: + - linux + - windows + - darwin + +archives: + - format: tar.gz + # this name template makes the OS and Arch compatible with the results of `uname`. + name_template: >- + {{ .ProjectName }}_ + {{- title .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else if eq .Arch "386" }}i386 + {{- else }}{{ .Arch }}{{ end }} + {{- if .Arm }}v{{ .Arm }}{{ end }} + # use zip for windows archives + format_overrides: + - goos: windows + format: zip + +changelog: + sort: asc + filters: + exclude: + - "^docs:" + - "^test:" From 49bce0fdc86aa19cae5f6d47161dced91d410271 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 21 Oct 2023 14:30:37 +0530 Subject: [PATCH 136/183] tidy go mod --- go.mod | 18 ++++++------------ go.sum | 33 ++++++++++----------------------- 2 files changed, 16 insertions(+), 35 deletions(-) diff --git a/go.mod b/go.mod index e1258a425..c5f6ba3f3 100644 --- a/go.mod +++ b/go.mod @@ -5,23 +5,17 @@ go 1.20 require ( github.com/go-resty/resty/v2 v2.7.0 github.com/google/uuid v1.3.1 - github.com/jamesruan/sodium v1.0.14 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 - github.com/pkg/errors v0.9.1 + github.com/zalando/go-keyring v0.2.3 golang.org/x/crypto v0.14.0 ) require ( - git.schwanenlied.me/yawning/chacha20.git v0.0.0-20170904085104-e3b1f968fc63 // indirect - github.com/Yawning/chacha20 v0.0.0-20170904085104-e3b1f968fc63 // indirect - github.com/Yawning/poly1305 v0.0.0-20151107134637-dfc796fe731c // indirect github.com/alessio/shellescape v1.4.1 // indirect github.com/danieljoos/wincred v1.2.0 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect - github.com/zalando/go-keyring v0.2.3 // indirect - go.artemisc.eu/godium v0.0.0-20201014025004-f08c318a71b3 // indirect ) require ( @@ -29,21 +23,21 @@ require ( github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/kong/go-srp v0.0.0-20191210190804-cde1efa3c083 // indirect + github.com/kong/go-srp v0.0.0-20191210190804-cde1efa3c083 github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/spf13/afero v1.9.5 // indirect github.com/spf13/cast v1.5.1 // indirect - github.com/spf13/cobra v1.7.0 // indirect + github.com/spf13/cobra v1.7.0 github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.16.0 // indirect + github.com/spf13/viper v1.16.0 github.com/subosito/gotenv v1.6.0 // indirect - go.etcd.io/bbolt v1.3.7 // indirect + go.etcd.io/bbolt v1.3.7 golang.org/x/net v0.10.0 // indirect golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect + golang.org/x/term v0.13.0 golang.org/x/text v0.13.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 10f11fe85..914c2284a 100644 --- a/go.sum +++ b/go.sum @@ -36,14 +36,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.schwanenlied.me/yawning/chacha20.git v0.0.0-20170904085104-e3b1f968fc63 h1:bwZNsbw3qFbg6ox55HrA37nPmh+/wtJxZ7uWeiAdUUc= -git.schwanenlied.me/yawning/chacha20.git v0.0.0-20170904085104-e3b1f968fc63/go.mod h1:NYi4Ifd1g/YbhIDgDfw6t7QdsW4tofQWMX/+FiDtJWs= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Yawning/chacha20 v0.0.0-20170904085104-e3b1f968fc63 h1:I6/SJSN9wJMJ+ZyQaCHUlzoTA4ypU5Bb44YWR1wTY/0= -github.com/Yawning/chacha20 v0.0.0-20170904085104-e3b1f968fc63/go.mod h1:nf+Komq6fVP4SwmKEaVGxHTyQGKREVlwjQKpvOV39yE= -github.com/Yawning/poly1305 v0.0.0-20151107134637-dfc796fe731c h1:JD5KufmlwGs/wsQhS1HrKKDMXYhLmp4XQvgoAoua4A0= -github.com/Yawning/poly1305 v0.0.0-20151107134637-dfc796fe731c/go.mod h1:CkwFWTKoa4/jtX6RLTogyqTlMn4898oEkv6j2Ai50+I= github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -58,8 +52,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/danieljoos/wincred v1.2.0 h1:ozqKHaLK0W/ii4KVbbvluM91W2H3Sh0BncbUNPS7jLE= github.com/danieljoos/wincred v1.2.0/go.mod h1:FzQLLMKBFdvu+osBrnFODiv32YGwCfx0SkRa/eYHgec= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dchest/siphash v1.2.2/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -68,6 +62,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -113,6 +108,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -141,8 +137,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jamesruan/sodium v1.0.14 h1:JfOHobip/lUWouxHV3PwYwu3gsLewPrDrZXO3HuBzUU= -github.com/jamesruan/sodium v1.0.14/go.mod h1:GK2+LACf7kuVQ9k7Irk0MB2B65j5rVqkz+9ylGIggZk= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -150,8 +144,10 @@ github.com/kong/go-srp v0.0.0-20191210190804-cde1efa3c083 h1:Y7nibF/3Ivmk+S4Q+Kz github.com/kong/go-srp v0.0.0-20191210190804-cde1efa3c083/go.mod h1:Zde5RRLiH8/2zEXQDHX5W0dOOTxkemzrXMhHVfxTtTA= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -165,12 +161,13 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= @@ -186,6 +183,7 @@ github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -193,6 +191,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= @@ -202,8 +201,6 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/zalando/go-keyring v0.2.3 h1:v9CUu9phlABObO4LPWycf+zwMG7nlbb3t/B5wa97yms= github.com/zalando/go-keyring v0.2.3/go.mod h1:HL4k+OXQfJUWaMnqyuSOc0drfGPX2b51Du6K+MRgZMk= -go.artemisc.eu/godium v0.0.0-20201014025004-f08c318a71b3 h1:UB0aMKbtlmdvZ3W1C3U3wFnXcS8tNTNksXF968gB3tc= -go.artemisc.eu/godium v0.0.0-20201014025004-f08c318a71b3/go.mod h1:Pav9C606SsOk8r0N+Ow8PQwA+C9WontLIGzfIFgPj9A= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -217,13 +214,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -350,15 +342,9 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -514,6 +500,7 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= From 350d8ed2b010d8f2751dea4dd20156e994ef21b8 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 21 Oct 2023 14:54:13 +0530 Subject: [PATCH 137/183] Add project name --- .goreleaser.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index d3fb70f3b..47dbe1eb9 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -5,7 +5,7 @@ # Feel free to remove those if you don't want/need to use them. # yaml-language-server: $schema=https://goreleaser.com/static/schema.json # vim: set ts=2 sw=2 tw=0 fo=cnqoj - +project_name: ente-cli before: hooks: # You may remove this if you don't use go modules. From e2007bb7e2f8814e3879148b6eeec5f7a425a28b Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 21 Oct 2023 14:56:13 +0530 Subject: [PATCH 138/183] Update module name --- cmd/account.go | 4 ++-- cmd/root.go | 2 +- go.mod | 2 +- internal/crypto/crypto_libsodium.go | 2 +- internal/promt.go | 2 +- main.go | 12 ++++++------ pkg/account.go | 8 ++++---- pkg/bolt_store.go | 4 ++-- pkg/cli.go | 4 ++-- pkg/disk.go | 4 ++-- pkg/download.go | 8 ++++---- pkg/mapper/photo.go | 12 ++++++------ pkg/model/account.go | 2 +- pkg/model/enc_string.go | 4 ++-- pkg/model/remote.go | 2 +- pkg/remote_sync.go | 6 +++--- pkg/remote_to_disk_album.go | 4 ++-- pkg/remote_to_disk_file.go | 8 ++++---- pkg/secrets/key_holder.go | 8 ++++---- pkg/secrets/secret.go | 2 +- pkg/sign_in.go | 10 +++++----- pkg/store.go | 2 +- pkg/sync.go | 6 +++--- 23 files changed, 59 insertions(+), 59 deletions(-) diff --git a/cmd/account.go b/cmd/account.go index 5704a4de7..72e719e62 100644 --- a/cmd/account.go +++ b/cmd/account.go @@ -1,10 +1,10 @@ package cmd import ( - "cli-go/internal/api" - "cli-go/pkg/model" "context" "fmt" + "github.com/ente-io/cli/internal/api" + "github.com/ente-io/cli/pkg/model" "github.com/spf13/cobra" ) diff --git a/cmd/root.go b/cmd/root.go index 8001465ba..2dca05fac 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,8 +1,8 @@ package cmd import ( - "cli-go/pkg" "fmt" + "github.com/ente-io/cli/pkg" "os" "runtime" diff --git a/go.mod b/go.mod index c5f6ba3f3..090185f64 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module cli-go +module github.com/ente-io/cli go 1.20 diff --git a/internal/crypto/crypto_libsodium.go b/internal/crypto/crypto_libsodium.go index 2c777b2c4..a7c193c99 100644 --- a/internal/crypto/crypto_libsodium.go +++ b/internal/crypto/crypto_libsodium.go @@ -2,8 +2,8 @@ package crypto import ( "bufio" - "cli-go/utils/encoding" "errors" + "github.com/ente-io/cli/utils/encoding" "golang.org/x/crypto/nacl/box" "golang.org/x/crypto/nacl/secretbox" "io" diff --git a/internal/promt.go b/internal/promt.go index e616934b3..527c149ba 100644 --- a/internal/promt.go +++ b/internal/promt.go @@ -1,9 +1,9 @@ package internal import ( - "cli-go/internal/api" "errors" "fmt" + "github.com/ente-io/cli/internal/api" "log" "os" diff --git a/main.go b/main.go index c46572e37..e6b3d44a4 100644 --- a/main.go +++ b/main.go @@ -1,13 +1,13 @@ package main import ( - "cli-go/cmd" - "cli-go/internal" - "cli-go/internal/api" - "cli-go/pkg" - "cli-go/pkg/secrets" - "cli-go/utils/constants" "fmt" + "github.com/ente-io/cli/cmd" + "github.com/ente-io/cli/internal" + "github.com/ente-io/cli/internal/api" + "github.com/ente-io/cli/pkg" + "github.com/ente-io/cli/pkg/secrets" + "github.com/ente-io/cli/utils/constants" "log" "strings" ) diff --git a/pkg/account.go b/pkg/account.go index 7ba782738..720b54bc0 100644 --- a/pkg/account.go +++ b/pkg/account.go @@ -1,13 +1,13 @@ package pkg import ( - "cli-go/internal" - "cli-go/internal/api" - "cli-go/pkg/model" - "cli-go/utils/encoding" "context" "encoding/json" "fmt" + "github.com/ente-io/cli/internal" + "github.com/ente-io/cli/internal/api" + "github.com/ente-io/cli/pkg/model" + "github.com/ente-io/cli/utils/encoding" "log" bolt "go.etcd.io/bbolt" diff --git a/pkg/bolt_store.go b/pkg/bolt_store.go index b13c3e28b..0561c7442 100644 --- a/pkg/bolt_store.go +++ b/pkg/bolt_store.go @@ -1,10 +1,10 @@ package pkg import ( - "cli-go/pkg/model" - "cli-go/utils/encoding" "context" "fmt" + "github.com/ente-io/cli/pkg/model" + "github.com/ente-io/cli/utils/encoding" ) func boltAEKey(entry *model.AlbumFileEntry) []byte { diff --git a/pkg/cli.go b/pkg/cli.go index 5096df3e1..d79e4138f 100644 --- a/pkg/cli.go +++ b/pkg/cli.go @@ -1,9 +1,9 @@ package pkg import ( - "cli-go/internal/api" - "cli-go/pkg/secrets" "fmt" + "github.com/ente-io/cli/internal/api" + "github.com/ente-io/cli/pkg/secrets" bolt "go.etcd.io/bbolt" "log" "os" diff --git a/pkg/disk.go b/pkg/disk.go index 2a81d4976..cae694f24 100644 --- a/pkg/disk.go +++ b/pkg/disk.go @@ -1,10 +1,10 @@ package pkg import ( - "cli-go/pkg/model" - "cli-go/pkg/model/export" "encoding/json" "errors" + "github.com/ente-io/cli/pkg/model" + "github.com/ente-io/cli/pkg/model/export" "io" "os" "strings" diff --git a/pkg/download.go b/pkg/download.go index 3bd39cb04..7a643b662 100644 --- a/pkg/download.go +++ b/pkg/download.go @@ -2,12 +2,12 @@ package pkg import ( "archive/zip" - "cli-go/internal/crypto" - "cli-go/pkg/model" - "cli-go/utils" - "cli-go/utils/encoding" "context" "fmt" + "github.com/ente-io/cli/internal/crypto" + "github.com/ente-io/cli/pkg/model" + "github.com/ente-io/cli/utils" + "github.com/ente-io/cli/utils/encoding" "io" "log" "os" diff --git a/pkg/mapper/photo.go b/pkg/mapper/photo.go index d213864cd..fa55ffae5 100644 --- a/pkg/mapper/photo.go +++ b/pkg/mapper/photo.go @@ -1,15 +1,15 @@ package mapper import ( - "cli-go/internal/api" - eCrypto "cli-go/internal/crypto" - "cli-go/pkg/model" - "cli-go/pkg/model/export" - "cli-go/pkg/secrets" - "cli-go/utils/encoding" "context" "encoding/json" "errors" + "github.com/ente-io/cli/internal/api" + eCrypto "github.com/ente-io/cli/internal/crypto" + "github.com/ente-io/cli/pkg/model" + "github.com/ente-io/cli/pkg/model/export" + "github.com/ente-io/cli/pkg/secrets" + "github.com/ente-io/cli/utils/encoding" "log" ) diff --git a/pkg/model/account.go b/pkg/model/account.go index 9f171c4f6..7e18a9f66 100644 --- a/pkg/model/account.go +++ b/pkg/model/account.go @@ -1,8 +1,8 @@ package model import ( - "cli-go/internal/api" "fmt" + "github.com/ente-io/cli/internal/api" ) type Account struct { diff --git a/pkg/model/enc_string.go b/pkg/model/enc_string.go index ebbbb891a..be6a6b939 100644 --- a/pkg/model/enc_string.go +++ b/pkg/model/enc_string.go @@ -1,8 +1,8 @@ package model import ( - "cli-go/internal/crypto" - "cli-go/utils/encoding" + "github.com/ente-io/cli/internal/crypto" + "github.com/ente-io/cli/utils/encoding" "log" ) diff --git a/pkg/model/remote.go b/pkg/model/remote.go index 32fb122cd..186edfb43 100644 --- a/pkg/model/remote.go +++ b/pkg/model/remote.go @@ -1,8 +1,8 @@ package model import ( - "cli-go/pkg/model/export" "fmt" + "github.com/ente-io/cli/pkg/model/export" "sort" "time" ) diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index 0b1886473..281afc329 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -1,12 +1,12 @@ package pkg import ( - "cli-go/pkg/mapper" - "cli-go/pkg/model" - "cli-go/utils/encoding" "context" "encoding/json" "fmt" + "github.com/ente-io/cli/pkg/mapper" + "github.com/ente-io/cli/pkg/model" + "github.com/ente-io/cli/utils/encoding" "log" "strconv" "time" diff --git a/pkg/remote_to_disk_album.go b/pkg/remote_to_disk_album.go index cf8c57434..cefb734a3 100644 --- a/pkg/remote_to_disk_album.go +++ b/pkg/remote_to_disk_album.go @@ -1,11 +1,11 @@ package pkg import ( - "cli-go/pkg/model" - "cli-go/pkg/model/export" "context" "encoding/json" "fmt" + "github.com/ente-io/cli/pkg/model" + "github.com/ente-io/cli/pkg/model/export" "log" "os" "strings" diff --git a/pkg/remote_to_disk_file.go b/pkg/remote_to_disk_file.go index a1c861414..51e8ef43e 100644 --- a/pkg/remote_to_disk_file.go +++ b/pkg/remote_to_disk_file.go @@ -1,14 +1,14 @@ package pkg import ( - "cli-go/pkg/mapper" - "cli-go/pkg/model" - "cli-go/pkg/model/export" - "cli-go/utils" "context" "encoding/json" "errors" "fmt" + "github.com/ente-io/cli/pkg/mapper" + "github.com/ente-io/cli/pkg/model" + "github.com/ente-io/cli/pkg/model/export" + "github.com/ente-io/cli/utils" "log" "os" "path/filepath" diff --git a/pkg/secrets/key_holder.go b/pkg/secrets/key_holder.go index dc82c72d1..37133606b 100644 --- a/pkg/secrets/key_holder.go +++ b/pkg/secrets/key_holder.go @@ -1,12 +1,12 @@ package secrets import ( - "cli-go/internal/api" - eCrypto "cli-go/internal/crypto" - "cli-go/pkg/model" - "cli-go/utils/encoding" "context" "fmt" + "github.com/ente-io/cli/internal/api" + eCrypto "github.com/ente-io/cli/internal/crypto" + "github.com/ente-io/cli/pkg/model" + "github.com/ente-io/cli/utils/encoding" ) type KeyHolder struct { diff --git a/pkg/secrets/secret.go b/pkg/secrets/secret.go index 49b61a232..438b2a8ad 100644 --- a/pkg/secrets/secret.go +++ b/pkg/secrets/secret.go @@ -1,10 +1,10 @@ package secrets import ( - "cli-go/utils/constants" "crypto/rand" "errors" "fmt" + "github.com/ente-io/cli/utils/constants" "log" "os" diff --git a/pkg/sign_in.go b/pkg/sign_in.go index d8b21f951..1e8c96118 100644 --- a/pkg/sign_in.go +++ b/pkg/sign_in.go @@ -1,13 +1,13 @@ package pkg import ( - "cli-go/internal" - "cli-go/internal/api" - eCrypto "cli-go/internal/crypto" - "cli-go/pkg/model" - "cli-go/utils/encoding" "context" "fmt" + "github.com/ente-io/cli/internal" + "github.com/ente-io/cli/internal/api" + eCrypto "github.com/ente-io/cli/internal/crypto" + "github.com/ente-io/cli/pkg/model" + "github.com/ente-io/cli/utils/encoding" "log" "github.com/kong/go-srp" diff --git a/pkg/store.go b/pkg/store.go index 18d67547e..aa866d595 100644 --- a/pkg/store.go +++ b/pkg/store.go @@ -1,9 +1,9 @@ package pkg import ( - "cli-go/pkg/model" "context" "fmt" + "github.com/ente-io/cli/pkg/model" "log" "strconv" "time" diff --git a/pkg/sync.go b/pkg/sync.go index 2fb0f6c01..d54ef0840 100644 --- a/pkg/sync.go +++ b/pkg/sync.go @@ -1,12 +1,12 @@ package pkg import ( - "cli-go/internal" - "cli-go/internal/api" - "cli-go/pkg/model" "context" "encoding/base64" "fmt" + "github.com/ente-io/cli/internal" + "github.com/ente-io/cli/internal/api" + "github.com/ente-io/cli/pkg/model" "log" bolt "go.etcd.io/bbolt" From 8ccac7b634e79462a6f8317a2ee8c1fed271269c Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sun, 22 Oct 2023 16:51:10 +0530 Subject: [PATCH 139/183] Add retry attempt in case of download failures --- internal/api/client.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/internal/api/client.go b/internal/api/client.go index f392fc5a2..b2d76b211 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -2,7 +2,9 @@ package api import ( "context" + "fmt" "github.com/go-resty/resty/v2" + "time" ) const ( @@ -66,8 +68,18 @@ func NewClient(p Params) *Client { enteAPI.SetBaseURL(EnteAPIEndpoint) } return &Client{ - restClient: enteAPI, - downloadClient: resty.New(), + restClient: enteAPI, + downloadClient: resty.New(). + SetRetryCount(3). + SetRetryWaitTime(5 * time.Second). + SetRetryMaxWaitTime(10 * time.Second). + AddRetryCondition(func(r *resty.Response, err error) bool { + shouldRetry := r.StatusCode() == 429 || r.StatusCode() > 500 + if shouldRetry { + fmt.Println(fmt.Printf("retrying download due to %d code", r.StatusCode())) + } + return shouldRetry + }), } } From 9b5edd1599fe5f4584ab89370062ff7721bcb5c0 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 23 Oct 2023 21:42:23 +0530 Subject: [PATCH 140/183] Add GithubAction for release --- .github/workflows/release.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..0d52c7075 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,28 @@ +name: Release + +on: + push: + tags: + - 'v*.*.*' # This will run the workflow when you push a new tag in the format v0.0.0 + +jobs: + goreleaser: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + fetch-depth: 0 # Important to ensure that GoReleaser works correctly + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.17 # You can adjust the Go version here + + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v2 + with: + version: latest + args: release --rm-dist + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Use the provided GITHUB_TOKEN secret From 31a66af0d6cf019ad287fa7217c2eb8cbf2ac651 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 23 Oct 2023 21:44:28 +0530 Subject: [PATCH 141/183] Update Github action --- .github/workflows/release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0d52c7075..ff857c9a7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,9 +1,11 @@ name: Release on: + # allow manual run push: tags: - 'v*.*.*' # This will run the workflow when you push a new tag in the format v0.0.0 + - 'v*.*.*-beta.*' jobs: goreleaser: From da9402de729f289e5acece0ffbd896104e71d6f2 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 23 Oct 2023 21:45:07 +0530 Subject: [PATCH 142/183] Change version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 2dca05fac..ada8a9161 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -11,7 +11,7 @@ import ( "github.com/spf13/cobra" ) -const AppVersion = "0.0.1" +const AppVersion = "0.1.0-beta" var ctrl *pkg.ClICtrl From 2320d56ee319c316e32b6ea1585814df2a3a9a87 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 23 Oct 2023 21:47:56 +0530 Subject: [PATCH 143/183] Update github action --- .github/workflows/release.yml | 2 +- cmd/root.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ff857c9a7..bc56ab764 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.17 # You can adjust the Go version here + go-version: 1.20 # You can adjust the Go version here - name: Run GoReleaser uses: goreleaser/goreleaser-action@v2 diff --git a/cmd/root.go b/cmd/root.go index ada8a9161..a3259e864 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -11,7 +11,7 @@ import ( "github.com/spf13/cobra" ) -const AppVersion = "0.1.0-beta" +const AppVersion = "0.1.1-beta" var ctrl *pkg.ClICtrl From bdacbd66a038584164ba8b46901cb2f26e3a8430 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Tue, 24 Oct 2023 08:39:48 +0530 Subject: [PATCH 144/183] update action --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bc56ab764..f9d39543e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,14 +12,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 # Important to ensure that GoReleaser works correctly - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v4 with: - go-version: 1.20 # You can adjust the Go version here + go-version: '1.20' # You can adjust the Go version here - name: Run GoReleaser uses: goreleaser/goreleaser-action@v2 From f0bf5c42f3dd9b539a45372539d24a5998acb667 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Tue, 24 Oct 2023 08:48:06 +0530 Subject: [PATCH 145/183] Update goreleaser --- .github/workflows/release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f9d39543e..d1a65e78b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,8 +22,9 @@ jobs: go-version: '1.20' # You can adjust the Go version here - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v2 + uses: goreleaser/goreleaser-action@v5 with: + distribution: goreleaser version: latest args: release --rm-dist env: From 0918412b35439ab0fba279b256e4276f9a37dee7 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Tue, 24 Oct 2023 12:03:51 +0530 Subject: [PATCH 146/183] imporve error --- internal/promt.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/promt.go b/internal/promt.go index 527c149ba..5c709ae2e 100644 --- a/internal/promt.go +++ b/internal/promt.go @@ -80,20 +80,21 @@ func GetExportDir() string { for { exportDir, err := GetUserInput("Enter export directory") if err != nil { + log.Printf("invalid export directory input: %s\n", err) return "" } if exportDir == "" { - fmt.Printf("invalid export directory: %s\n", err) + log.Printf("invalid export directory: %s\n", err) continue } exportDir, err = ResolvePath(exportDir) if err != nil { - fmt.Printf("invalid export directory: %s\n", err) + log.Printf("invalid export directory: %s\n", err) continue } _, err = ValidateDirForWrite(exportDir) if err != nil { - fmt.Printf("invalid export directory: %s\n", err) + log.Printf("invalid export directory: %s\n", err) continue } From 29925a910c8734fee36c7fe8e4eeb30ab7b9aab1 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Tue, 24 Oct 2023 18:05:37 +0530 Subject: [PATCH 147/183] Fix file path --- pkg/remote_to_disk_album.go | 3 ++- pkg/remote_to_disk_file.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/remote_to_disk_album.go b/pkg/remote_to_disk_album.go index cefb734a3..6253a3e4c 100644 --- a/pkg/remote_to_disk_album.go +++ b/pkg/remote_to_disk_album.go @@ -38,6 +38,7 @@ func (c *ClICtrl) createLocalFolderForRemoteAlbums(ctx context.Context, account continue } metaByID := albumIDToMetaMap[album.ID] + if metaByID != nil { if strings.EqualFold(metaByID.AlbumName, album.AlbumName) { //log.Printf("Skipping album %s as it already exists", album.AlbumName) @@ -45,7 +46,7 @@ func (c *ClICtrl) createLocalFolderForRemoteAlbums(ctx context.Context, account } } - albumFolderName := album.AlbumName + albumFolderName := filepath.Clean(album.AlbumName) albumID := album.ID if _, ok := folderToMetaMap[albumFolderName]; ok { diff --git a/pkg/remote_to_disk_file.go b/pkg/remote_to_disk_file.go index 51e8ef43e..ced04945a 100644 --- a/pkg/remote_to_disk_file.go +++ b/pkg/remote_to_disk_file.go @@ -116,7 +116,7 @@ func (c *ClICtrl) downloadEntry(ctx context.Context, fileDiskMetadata := mapper.MapRemoteFileToDiskMetadata(file) // Get the extension extension := filepath.Ext(fileDiskMetadata.Title) - baseFileName := strings.TrimSuffix(filepath.Base(fileDiskMetadata.Title), extension) + baseFileName := strings.TrimSuffix(filepath.Clean(filepath.Base(fileDiskMetadata.Title)), extension) potentialDiskFileName := fmt.Sprintf("%s%s.json", baseFileName, extension) count := 1 for diskInfo.IsMetaFileNamePresent(potentialDiskFileName) { From 104bdfb5a4f19c19dad5748c264a17f466258e90 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Tue, 24 Oct 2023 18:11:46 +0530 Subject: [PATCH 148/183] Add prompt to export after account add --- pkg/account.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/account.go b/pkg/account.go index 720b54bc0..2fd505e70 100644 --- a/pkg/account.go +++ b/pkg/account.go @@ -73,6 +73,7 @@ func (c *ClICtrl) AddAccount(cxt context.Context) { return } else { fmt.Println("Account added successfully") + fmt.Println("run `ente-cli export` to initiate export of your account data") } } From 405d70b43f02693c13a871bee2a7e47b975cc654 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 25 Oct 2023 08:28:54 +0530 Subject: [PATCH 149/183] Use .ente-cli as config file path --- main.go | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index e6b3d44a4..f02523f4b 100644 --- a/main.go +++ b/main.go @@ -9,11 +9,17 @@ import ( "github.com/ente-io/cli/pkg/secrets" "github.com/ente-io/cli/utils/constants" "log" + "os" + "path/filepath" "strings" ) func main() { - cliDBPath := "" + cliDBPath, err := GetCLIConfigPath() + if err != nil { + log.Fatalf("Could not create cli config path\n%v\n", err) + } + db, err := pkg.GetDB(fmt.Sprintf("%sente-cli.db", cliDBPath)) if secrets.IsRunningInContainer() { cliDBPath = constants.CliDataPath _, err := internal.ValidateDirForWrite(cliDBPath) @@ -21,7 +27,7 @@ func main() { log.Fatalf("Please mount a volume to %s to persist cli data\n%v\n", cliDBPath, err) } } - db, err := pkg.GetDB(fmt.Sprintf("%sente-cli.db", cliDBPath)) + if err != nil { if strings.Contains(err.Error(), "timeout") { log.Fatalf("Please close all other instances of the cli and try again\n%v\n", err) @@ -48,3 +54,28 @@ func main() { }() cmd.Execute(&ctrl) } + +// GetCLIConfigPath returns the path to the .ente-cli folder and creates it if it doesn't exist. +func GetCLIConfigPath() (string, error) { + if os.Getenv("ENTE_CLI_CONFIG_PATH") != "" { + return os.Getenv("ENTE_CLI_CONFIG_PATH"), nil + } + // Get the user's home directory + homeDir, err := os.UserHomeDir() + if err != nil { + return "", err + } + + // Create the path for the .ente-cli folder + cliDBPath := filepath.Join(homeDir, ".ente-cli") + + // Check if the folder already exists, if not, create it + if _, err := os.Stat(cliDBPath); os.IsNotExist(err) { + err := os.MkdirAll(cliDBPath, 0755) + if err != nil { + return "", err + } + } + + return cliDBPath, nil +} From b374fd23e984d24652f0334f7059b27cbb9e09c3 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 25 Oct 2023 08:29:23 +0530 Subject: [PATCH 150/183] Remove redundant log --- pkg/cli.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/cli.go b/pkg/cli.go index d79e4138f..3d1ce4914 100644 --- a/pkg/cli.go +++ b/pkg/cli.go @@ -5,7 +5,6 @@ import ( "github.com/ente-io/cli/internal/api" "github.com/ente-io/cli/pkg/secrets" bolt "go.etcd.io/bbolt" - "log" "os" "path/filepath" ) @@ -26,7 +25,6 @@ func (c *ClICtrl) Init() error { return err } } - log.Printf("Using temp folder %s", tempPath) c.tempFolder = tempPath return c.DB.Update(func(tx *bolt.Tx) error { _, err := tx.CreateBucketIfNotExists([]byte(AccBucket)) From d3c07e0a177d0b20bff71f09fa309e133505c5d4 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 25 Oct 2023 17:22:22 +0530 Subject: [PATCH 151/183] Rename ente-cli to ente --- .goreleaser.yaml | 2 +- cmd/root.go | 2 +- cmd/version.go | 2 +- main.go | 3 +-- pkg/account.go | 2 +- pkg/cli.go | 2 +- pkg/secrets/secret.go | 10 ++++++++-- release.sh | 4 ++-- 8 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 47dbe1eb9..892c2ccde 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -5,7 +5,7 @@ # Feel free to remove those if you don't want/need to use them. # yaml-language-server: $schema=https://goreleaser.com/static/schema.json # vim: set ts=2 sw=2 tw=0 fo=cnqoj -project_name: ente-cli +project_name: ente before: hooks: # You may remove this if you don't use go modules. diff --git a/cmd/root.go b/cmd/root.go index a3259e864..277152167 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -17,7 +17,7 @@ var ctrl *pkg.ClICtrl // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ - Use: "ente-cli", + Use: "ente", Short: "CLI tool for exporting your photos from ente.io", Long: `Start by creating a config file in your home directory:`, // Uncomment the following line if your bare application diff --git a/cmd/version.go b/cmd/version.go index cb4d1cf47..9d5194bc7 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -12,7 +12,7 @@ var versionCmd = &cobra.Command{ Short: "Prints the current version", Long: ``, Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("ente-cli version %s\n", AppVersion) + fmt.Printf("Version %s\n", AppVersion) }, } diff --git a/main.go b/main.go index f02523f4b..254b39b9d 100644 --- a/main.go +++ b/main.go @@ -66,8 +66,7 @@ func GetCLIConfigPath() (string, error) { return "", err } - // Create the path for the .ente-cli folder - cliDBPath := filepath.Join(homeDir, ".ente-cli") + cliDBPath := filepath.Join(homeDir, ".ente") // Check if the folder already exists, if not, create it if _, err := os.Stat(cliDBPath); os.IsNotExist(err) { diff --git a/pkg/account.go b/pkg/account.go index 2fd505e70..df45f3239 100644 --- a/pkg/account.go +++ b/pkg/account.go @@ -73,7 +73,7 @@ func (c *ClICtrl) AddAccount(cxt context.Context) { return } else { fmt.Println("Account added successfully") - fmt.Println("run `ente-cli export` to initiate export of your account data") + fmt.Println("run `ente export` to initiate export of your account data") } } diff --git a/pkg/cli.go b/pkg/cli.go index 3d1ce4914..cc9c39937 100644 --- a/pkg/cli.go +++ b/pkg/cli.go @@ -17,7 +17,7 @@ type ClICtrl struct { } func (c *ClICtrl) Init() error { - tempPath := filepath.Join(os.TempDir(), "ente-cli-download") + tempPath := filepath.Join(os.TempDir(), "ente-download") // create temp folder if not exists if _, err := os.Stat(tempPath); os.IsNotExist(err) { err = os.Mkdir(tempPath, 0755) diff --git a/pkg/secrets/secret.go b/pkg/secrets/secret.go index 438b2a8ad..1ebd13a1d 100644 --- a/pkg/secrets/secret.go +++ b/pkg/secrets/secret.go @@ -17,9 +17,15 @@ func IsRunningInContainer() bool { } return true } + +const ( + secretService = "ente" + secretUser = "ente-cli-user" +) + func GetOrCreateClISecret() []byte { // get password - secret, err := keyring.Get("ente-cli-cli", "ghost") + secret, err := keyring.Get(secretService, secretUser) if err != nil { if !errors.Is(err, keyring.ErrNotFound) { if IsRunningInContainer() { @@ -34,7 +40,7 @@ func GetOrCreateClISecret() []byte { log.Fatal(fmt.Errorf("error generating key: %w", err)) } secret = string(key) - keySetErr := keyring.Set("ente-cli-cli", "ghost", string(secret)) + keySetErr := keyring.Set(secretService, secretUser, string(secret)) if keySetErr != nil { log.Fatal(fmt.Errorf("error setting password in keyring: %w", keySetErr)) } diff --git a/release.sh b/release.sh index 7c5047385..8e2deba0f 100755 --- a/release.sh +++ b/release.sh @@ -20,11 +20,11 @@ do export GOARCH="$ARCH" # Set the output binary name to "ente-cli" for the current OS and architecture - BINARY_NAME="ente-cli-$OS-$ARCH" + BINARY_NAME="ente-$OS-$ARCH" # Add .exe extension for Windows if [ "$OS" == "windows" ]; then - BINARY_NAME="ente-cli-$OS-$ARCH.exe" + BINARY_NAME="ente-$OS-$ARCH.exe" fi # Build the binary and place it in the "bin" directory From 0b1c98e2f9415bcb6e3ce9ecfa176e8d8265127d Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 25 Oct 2023 17:23:38 +0530 Subject: [PATCH 152/183] Update version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 277152167..2453f395d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -11,7 +11,7 @@ import ( "github.com/spf13/cobra" ) -const AppVersion = "0.1.1-beta" +const AppVersion = "0.1.3" var ctrl *pkg.ClICtrl From 66d494b48052c4567e00ef6123eb0309ae0df6cb Mon Sep 17 00:00:00 2001 From: Vishnu Mohandas Date: Wed, 25 Oct 2023 17:24:36 +0530 Subject: [PATCH 153/183] Update README.md --- README.md | 80 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 45 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index e005a8dd9..b76ad24d9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# cli for exporting ente.io data +# Command Line Utility for exporting data from [Ente](https://ente.io) ## Install @@ -7,70 +7,80 @@ You can either download the binary from the [release page](https://github.com/en ### Build from source ```shell - go build -o "bin/ente-cli" main.go + go build -o "bin/ente" main.go ``` ### Getting Started Run the help command to see all available commands. ```shell -ente-cli --help +ente --help ``` #### Accounts -If you wish, you can add multiple accounts (your own and your family members) and export all using this tool. -* Add an account - ```shell - ente-cli account add - ``` +If you wish, you can add multiple accounts (your own and that of your family members) and export all data using this tool. -* List accounts - ```shell - ente-cli account list - ``` +##### Add an account +```shell +ente account add +``` + +##### List accounts +```shell +ente account list +``` -* Change export directory - ```shell - ente-cli account update --email yourEmail@example.com --dir ~/photos - ``` +##### Change export directory +```shell +ente account update --email email@domain.com --dir ~/photos +``` ### Export -* Start export - ```shell - ente-cli export - ``` +##### Start export +```shell +ente export +``` + +--- ## Docker +If you fancy Docker, you can also run the CLI within a container. + ### Configure + Modify the `docker-compose.yml` and add volume. ``cli-data`` volume is mandatory, you can add more volumes for your export directory. - * Build the docker image - ```shell - docker build -t ente-cli:latest . - ``` - * Start the container in detached mode - ```bash - docker-compose up -d - ``` -exec into the container + +Build the docker image ```shell - docker-compose exec ente-cli /bin/sh +docker build -t ente:latest . +``` + +Start the container in detached mode +```bash +docker-compose up -d +``` + +`exec` into the container +```shell +docker-compose exec ente /bin/sh ``` -#### How to directly execute the command +#### Directly executing commands - ```shell - docker run -it --rm ente-cli:latest ls - ``` +```shell +docker run -it --rm ente:latest ls +``` +--- ## Releases Run the release script to build the binary and run it. ```shell - ./release.sh +./release.sh ``` From 4c09bf0162de7785749c7057ef0a57dccba1db99 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 25 Oct 2023 17:28:48 +0530 Subject: [PATCH 154/183] Update goreleaser --- .goreleaser.yaml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 892c2ccde..0e99c1cc8 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -21,20 +21,20 @@ builds: - windows - darwin -archives: - - format: tar.gz - # this name template makes the OS and Arch compatible with the results of `uname`. - name_template: >- - {{ .ProjectName }}_ - {{- title .Os }}_ - {{- if eq .Arch "amd64" }}x86_64 - {{- else if eq .Arch "386" }}i386 - {{- else }}{{ .Arch }}{{ end }} - {{- if .Arm }}v{{ .Arm }}{{ end }} - # use zip for windows archives - format_overrides: - - goos: windows - format: zip +#archives: +# - format: tar.gz +# # this name template makes the OS and Arch compatible with the results of `uname`. +# name_template: >- +# {{ .ProjectName }}_ +# {{- title .Os }}_ +# {{- if eq .Arch "amd64" }}x86_64 +# {{- else if eq .Arch "386" }}i386 +# {{- else }}{{ .Arch }}{{ end }} +# {{- if .Arm }}v{{ .Arm }}{{ end }} +# # use zip for windows archives +# format_overrides: +# - goos: windows +# format: zip changelog: sort: asc From 7a8de8fc4b3a9ed1ad542dd7e61df8d6a6892dcb Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 25 Oct 2023 17:56:53 +0530 Subject: [PATCH 155/183] Fix path --- .goreleaser.yaml | 42 ++++++++++++++++++++++++------------- cmd/root.go | 2 +- pkg/remote_to_disk_album.go | 2 +- 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 0e99c1cc8..870d6f76d 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -21,20 +21,34 @@ builds: - windows - darwin -#archives: -# - format: tar.gz -# # this name template makes the OS and Arch compatible with the results of `uname`. -# name_template: >- -# {{ .ProjectName }}_ -# {{- title .Os }}_ -# {{- if eq .Arch "amd64" }}x86_64 -# {{- else if eq .Arch "386" }}i386 -# {{- else }}{{ .Arch }}{{ end }} -# {{- if .Arm }}v{{ .Arm }}{{ end }} -# # use zip for windows archives -# format_overrides: -# - goos: windows -# format: zip +nfpms: + - package_name: ente + homepage: https://github.com/ente-io/cli + maintainer: ente.io + description: |- + Command Line Utility for exporting data from https://ente.io + formats: + - rpm + - deb + - apk + +sboms: + - artifacts: archive + +archives: + - format: tar.gz + # this name template makes the OS and Arch compatible with the results of `uname`. + name_template: >- + {{ .ProjectName }}_ + {{- title .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else if eq .Arch "386" }}i386 + {{- else }}{{ .Arch }}{{ end }} + {{- if .Arm }}v{{ .Arm }}{{ end }} + # use zip for windows archives + format_overrides: + - goos: windows + format: zip changelog: sort: asc diff --git a/cmd/root.go b/cmd/root.go index 2453f395d..1b39fa6ff 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -11,7 +11,7 @@ import ( "github.com/spf13/cobra" ) -const AppVersion = "0.1.3" +const AppVersion = "0.1.5" var ctrl *pkg.ClICtrl diff --git a/pkg/remote_to_disk_album.go b/pkg/remote_to_disk_album.go index 6253a3e4c..247d05499 100644 --- a/pkg/remote_to_disk_album.go +++ b/pkg/remote_to_disk_album.go @@ -59,7 +59,7 @@ func (c *ClICtrl) createLocalFolderForRemoteAlbums(ctx context.Context, account } } // Create album and meta folders if they don't exist - albumPath := filepath.Join(path, albumFolderName) + albumPath := filepath.Clean(filepath.Join(path, albumFolderName)) metaPath := filepath.Join(albumPath, ".meta") if metaByID == nil { log.Printf("Adding folder %s for album %s", albumFolderName, album.AlbumName) From ae13d8557bd9ae295e66ba1de58fdb044ba3da8e Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 25 Oct 2023 18:09:17 +0530 Subject: [PATCH 156/183] Install Syft in CI build machine --- .github/workflows/release.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d1a65e78b..834c21fdb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,6 +16,11 @@ jobs: with: fetch-depth: 0 # Important to ensure that GoReleaser works correctly + - name: Install latest Syft + run: | + wget $(curl -s https://api.github.com/repos/anchore/syft/releases/latest | grep 'browser_' | grep 'linux_amd64.rpm' | cut -d\" -f4) -O syft_latest_linux_amd64.rpm + sudo rpm -i syft_latest_linux_amd64.rpm + - name: Set up Go uses: actions/setup-go@v4 with: From 7d72c794421f3cb68040776035fa5a8802b31cf0 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 25 Oct 2023 18:09:28 +0530 Subject: [PATCH 157/183] Fixed bug in syncing on windows --- pkg/disk.go | 3 +-- pkg/remote_to_disk_album.go | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/disk.go b/pkg/disk.go index cae694f24..5dc9acbae 100644 --- a/pkg/disk.go +++ b/pkg/disk.go @@ -7,7 +7,6 @@ import ( "github.com/ente-io/cli/pkg/model/export" "io" "os" - "strings" ) const ( @@ -101,7 +100,7 @@ func readJSONFromFile(filePath string, data interface{}) error { func Move(source, destination string) error { err := os.Rename(source, destination) - if err != nil && strings.Contains(err.Error(), "cross-device link") { + if err != nil { return moveCrossDevice(source, destination) } return err diff --git a/pkg/remote_to_disk_album.go b/pkg/remote_to_disk_album.go index 247d05499..dc1dfeacb 100644 --- a/pkg/remote_to_disk_album.go +++ b/pkg/remote_to_disk_album.go @@ -47,6 +47,8 @@ func (c *ClICtrl) createLocalFolderForRemoteAlbums(ctx context.Context, account } albumFolderName := filepath.Clean(album.AlbumName) + // replace : with _ + albumFolderName = strings.ReplaceAll(albumFolderName, ":", "_") albumID := album.ID if _, ok := folderToMetaMap[albumFolderName]; ok { From d7288546bf76b33ba2eb4f024860f80cb81494ce Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 25 Oct 2023 18:09:42 +0530 Subject: [PATCH 158/183] Bump version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 1b39fa6ff..bea4a2217 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -11,7 +11,7 @@ import ( "github.com/spf13/cobra" ) -const AppVersion = "0.1.5" +const AppVersion = "0.1.6" var ctrl *pkg.ClICtrl From c2e624d919b570e4b88ec321bce085a2f9a82a14 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 25 Oct 2023 18:11:34 +0530 Subject: [PATCH 159/183] Fix CI order --- .github/workflows/release.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 834c21fdb..e5dca6923 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,16 +11,16 @@ jobs: goreleaser: runs-on: ubuntu-latest steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Important to ensure that GoReleaser works correctly - - name: Install latest Syft run: | wget $(curl -s https://api.github.com/repos/anchore/syft/releases/latest | grep 'browser_' | grep 'linux_amd64.rpm' | cut -d\" -f4) -O syft_latest_linux_amd64.rpm sudo rpm -i syft_latest_linux_amd64.rpm + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Important to ensure that GoReleaser works correctly + - name: Set up Go uses: actions/setup-go@v4 with: From fda7fe02a957035a25bb62a9f20d041f88fb5f41 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 30 Oct 2023 05:39:01 +0530 Subject: [PATCH 160/183] Delete encrypted file post download --- pkg/download.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/download.go b/pkg/download.go index 7a643b662..b09d44fce 100644 --- a/pkg/download.go +++ b/pkg/download.go @@ -37,6 +37,8 @@ func (c *ClICtrl) downloadAndDecrypt( if err != nil { log.Printf("Error decrypting file %d: %s", file.ID, err) return nil, model.ErrDecryption + } else { + _ = os.Remove(downloadPath) } return &decryptedPath, nil } From 8e9c43057d7818e9f123f989e3a3faa34eb85f5f Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 30 Oct 2023 08:01:25 +0530 Subject: [PATCH 161/183] Fix: Handle with file same name but different case --- pkg/disk.go | 37 ++++++++++++++++++++++++++++++------- pkg/remote_to_disk_file.go | 23 ++++++++++++----------- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/pkg/disk.go b/pkg/disk.go index 5dc9acbae..586c1f53a 100644 --- a/pkg/disk.go +++ b/pkg/disk.go @@ -3,10 +3,12 @@ package pkg import ( "encoding/json" "errors" + "fmt" "github.com/ente-io/cli/pkg/model" "github.com/ente-io/cli/pkg/model/export" "io" "os" + "strings" ) const ( @@ -30,7 +32,7 @@ func (a *albumDiskInfo) IsFilePresent(file model.RemoteFile) bool { } func (a *albumDiskInfo) IsFileNamePresent(fileName string) bool { - _, ok := (*a.FileNames)[fileName] + _, ok := (*a.FileNames)[strings.ToLower(fileName)] return ok } @@ -38,11 +40,17 @@ func (a *albumDiskInfo) AddEntry(metadata *export.DiskFileMetadata) error { if _, ok := (*a.FileIdToDiskFileMap)[metadata.Info.ID]; ok { return errors.New("fileID already present") } - if _, ok := (*a.MetaFileNameToDiskFileMap)[metadata.MetaFileName]; ok { + if _, ok := (*a.MetaFileNameToDiskFileMap)[strings.ToLower(metadata.MetaFileName)]; ok { return errors.New("fileName already present") } - (*a.MetaFileNameToDiskFileMap)[metadata.MetaFileName] = metadata + (*a.MetaFileNameToDiskFileMap)[strings.ToLower(metadata.MetaFileName)] = metadata (*a.FileIdToDiskFileMap)[metadata.Info.ID] = metadata + for _, filename := range metadata.Info.FileNames { + if _, ok := (*a.FileNames)[strings.ToLower(filename)]; ok { + return errors.New("fileName already present") + } + (*a.FileNames)[strings.ToLower(filename)] = true + } return nil } @@ -50,22 +58,37 @@ func (a *albumDiskInfo) RemoveEntry(metadata *export.DiskFileMetadata) error { if _, ok := (*a.FileIdToDiskFileMap)[metadata.Info.ID]; !ok { return errors.New("fileID not present") } - if _, ok := (*a.MetaFileNameToDiskFileMap)[metadata.MetaFileName]; !ok { + if _, ok := (*a.MetaFileNameToDiskFileMap)[strings.ToLower(metadata.MetaFileName)]; !ok { return errors.New("fileName not present") } - delete(*a.MetaFileNameToDiskFileMap, metadata.MetaFileName) + delete(*a.MetaFileNameToDiskFileMap, strings.ToLower(metadata.MetaFileName)) delete(*a.FileIdToDiskFileMap, metadata.Info.ID) for _, filename := range metadata.Info.FileNames { - delete(*a.FileNames, filename) + delete(*a.FileNames, strings.ToLower(filename)) } return nil } func (a *albumDiskInfo) IsMetaFileNamePresent(metaFileName string) bool { - _, ok := (*a.MetaFileNameToDiskFileMap)[metaFileName] + _, ok := (*a.MetaFileNameToDiskFileMap)[strings.ToLower(metaFileName)] return ok } +// GenerateUniqueFileName generates a unique file name. +func (a *albumDiskInfo) GenerateUniqueFileName(baseFileName, extension string) string { + fileName := fmt.Sprintf("%s%s", baseFileName, extension) + count := 1 + for a.IsFileNamePresent(strings.ToLower(fileName)) { + // separate the file name and extension + fileName = fmt.Sprintf("%s_%d%s", baseFileName, count, extension) + count++ + if !a.IsFileNamePresent(strings.ToLower(fileName)) { + break + } + } + return fileName +} + func (a *albumDiskInfo) GetDiskFileMetadata(file model.RemoteFile) *export.DiskFileMetadata { // check if file.ID is present diskFile, ok := (*a.FileIdToDiskFileMap)[file.ID] diff --git a/pkg/remote_to_disk_file.go b/pkg/remote_to_disk_file.go index ced04945a..388d7098d 100644 --- a/pkg/remote_to_disk_file.go +++ b/pkg/remote_to_disk_file.go @@ -128,11 +128,6 @@ func (c *ClICtrl) downloadEntry(ctx context.Context, break } } - fileDiskMetadata.MetaFileName = potentialDiskFileName - err = diskInfo.AddEntry(fileDiskMetadata) - if err != nil { - return err - } if file.IsLivePhoto() { imagePath, videoPath, err := UnpackLive(*decrypt) if err != nil { @@ -140,8 +135,8 @@ func (c *ClICtrl) downloadEntry(ctx context.Context, } imageExtn := filepath.Ext(imagePath) videoExtn := filepath.Ext(videoPath) - imageFileName := baseFileName + imageExtn - videoFileName := baseFileName + videoExtn + imageFileName := diskInfo.GenerateUniqueFileName(baseFileName, imageExtn) + videoFileName := diskInfo.GenerateUniqueFileName(baseFileName, videoExtn) imageFilePath := filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, imageFileName) videoFilePath := filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, videoFileName) // move the decrypt file to filePath @@ -156,7 +151,7 @@ func (c *ClICtrl) downloadEntry(ctx context.Context, fileDiskMetadata.AddFileName(imageFileName) fileDiskMetadata.AddFileName(videoFileName) } else { - fileName := baseFileName + extension + fileName := diskInfo.GenerateUniqueFileName(baseFileName, extension) filePath := filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, fileName) // move the decrypt file to filePath err = Move(*decrypt, filePath) @@ -166,6 +161,12 @@ func (c *ClICtrl) downloadEntry(ctx context.Context, fileDiskMetadata.AddFileName(fileName) } + fileDiskMetadata.MetaFileName = potentialDiskFileName + err = diskInfo.AddEntry(fileDiskMetadata) + if err != nil { + return err + } + err = writeJSONToFile(filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, ".meta", potentialDiskFileName), fileDiskMetadata) if err != nil { return err @@ -219,7 +220,7 @@ func readFilesMetadata(home string, albumMeta *export.AlbumMetadata) (*albumDisk } for _, entry := range albumFileEntries { if !entry.IsDir() { - claimedFileName[entry.Name()] = true + claimedFileName[strings.ToLower(entry.Name())] = true } } metaEntries, err := os.ReadDir(albumMetadataFolder) @@ -238,7 +239,7 @@ func readFilesMetadata(home string, albumMeta *export.AlbumMetadata) (*albumDisk } fileMetadataPath := filepath.Join(albumMetadataFolder, fileName) // Initialize as nil, will remain nil if JSON file is not found or not readable - result[fileName] = nil + result[strings.ToLower(fileName)] = nil // Read the JSON file if it exists var metaData export.DiskFileMetadata metaDataBytes, err := os.ReadFile(fileMetadataPath) @@ -247,7 +248,7 @@ func readFilesMetadata(home string, albumMeta *export.AlbumMetadata) (*albumDisk } if err := json.Unmarshal(metaDataBytes, &metaData); err == nil { metaData.MetaFileName = fileName - result[fileName] = &metaData + result[strings.ToLower(fileName)] = &metaData fileIdToMetadata[metaData.Info.ID] = &metaData } } From 9135c9181275721a0256f0d5e121d29cdcf0f792 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 30 Oct 2023 08:15:29 +0530 Subject: [PATCH 162/183] Log retry attempt --- internal/api/client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/api/client.go b/internal/api/client.go index b2d76b211..b2f164589 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -2,8 +2,8 @@ package api import ( "context" - "fmt" "github.com/go-resty/resty/v2" + "log" "time" ) @@ -76,7 +76,7 @@ func NewClient(p Params) *Client { AddRetryCondition(func(r *resty.Response, err error) bool { shouldRetry := r.StatusCode() == 429 || r.StatusCode() > 500 if shouldRetry { - fmt.Println(fmt.Printf("retrying download due to %d code", r.StatusCode())) + log.Printf("retrying download due to %d code", r.StatusCode()) } return shouldRetry }), From 6f11aa75b626c5af771557d0d74469390d06a718 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 30 Oct 2023 08:34:22 +0530 Subject: [PATCH 163/183] Auto retry export on collection failure --- pkg/model/errors.go | 10 +++++++++- pkg/model/remote.go | 3 --- pkg/sync.go | 29 +++++++++++++++++++++-------- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/pkg/model/errors.go b/pkg/model/errors.go index 7f041ef6e..d8f61d275 100644 --- a/pkg/model/errors.go +++ b/pkg/model/errors.go @@ -1,5 +1,13 @@ package model -import "errors" +import ( + "errors" + "strings" +) var ErrDecryption = errors.New("error while decrypting the file") + +func ShouldRetrySync(err error) bool { + return strings.Contains(err.Error(), "read tcp") || + strings.Contains(err.Error(), "dial tcp") +} diff --git a/pkg/model/remote.go b/pkg/model/remote.go index 186edfb43..75ff88c43 100644 --- a/pkg/model/remote.go +++ b/pkg/model/remote.go @@ -142,9 +142,6 @@ func (r *RemoteFile) GetModificationTime() time.Time { } func (r *RemoteFile) GetLatlong() *export.Location { - if r.ID == 10698020 { - fmt.Println("found 10698020") - } if r.PublicMetadata != nil { // check if lat and long key exists if lat, ok := r.PublicMetadata["lat"]; ok { diff --git a/pkg/sync.go b/pkg/sync.go index d54ef0840..6aee7cbb7 100644 --- a/pkg/sync.go +++ b/pkg/sync.go @@ -7,9 +7,9 @@ import ( "github.com/ente-io/cli/internal" "github.com/ente-io/cli/internal/api" "github.com/ente-io/cli/pkg/model" - "log" - bolt "go.etcd.io/bbolt" + "log" + "time" ) func (c *ClICtrl) Export() error { @@ -37,12 +37,23 @@ func (c *ClICtrl) Export() error { continue } log.Println("start sync") - err = c.SyncAccount(account) - if err != nil { - fmt.Printf("Error syncing account %s: %s\n", account.Email, err) - return err - } else { - log.Println("sync done") + retryCount := 0 + for { + err = c.SyncAccount(account) + if err != nil { + if model.ShouldRetrySync(err) && retryCount < 20 { + retryCount = retryCount + 1 + timeInSecond := time.Duration(retryCount*10) * time.Second + log.Printf("Connection err, waiting for %s before trying again", timeInSecond.String()) + time.Sleep(timeInSecond) + continue + } + fmt.Printf("Error syncing account %s: %s\n", account.Email, err) + return err + } else { + log.Println("sync done") + break + } } } @@ -63,10 +74,12 @@ func (c *ClICtrl) SyncAccount(account model.Account) error { err = c.fetchRemoteCollections(ctx) if err != nil { log.Printf("Error fetching collections: %s", err) + return err } err = c.fetchRemoteFiles(ctx) if err != nil { log.Printf("Error fetching files: %s", err) + return err } err = c.createLocalFolderForRemoteAlbums(ctx, account) if err != nil { From ece4a2b722a0656c596e556d02f4c0b5bc461485 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 30 Oct 2023 08:56:03 +0530 Subject: [PATCH 164/183] Add note about buildKit in the DockerFile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 522350be9..827269f5f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,7 @@ COPY go.sum . RUN go mod download COPY . . +# the --mount option requires BuildKit. Refer to https://docs.docker.com/go/buildkit/ to learn how to build images with BuildKit enabled RUN --mount=type=cache,target=/root/.cache/go-build \ go build -o ente-cli main.go From 5cbb6990032e134f728a933a6811024720b57de4 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 30 Oct 2023 09:02:55 +0530 Subject: [PATCH 165/183] Fix: Ignore remove err if file does not exist --- pkg/remote_to_disk_file.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/remote_to_disk_file.go b/pkg/remote_to_disk_file.go index 388d7098d..79352e119 100644 --- a/pkg/remote_to_disk_file.go +++ b/pkg/remote_to_disk_file.go @@ -184,12 +184,12 @@ func removeDiskFile(diskFileMeta *export.DiskFileMetadata, diskInfo *albumDiskIn // remove the file from disk log.Printf("Removing file %s from disk", diskFileMeta.MetaFileName) err := os.Remove(filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, ".meta", diskFileMeta.MetaFileName)) - if err != nil { + if err != nil && !os.IsNotExist(err) { return err } for _, fileName := range diskFileMeta.Info.FileNames { err = os.Remove(filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, fileName)) - if err != nil { + if err != nil && !os.IsNotExist(err) { return err } } From 94f2d5953ff142eade3c9af48a546f11a634e06d Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 30 Oct 2023 09:07:55 +0530 Subject: [PATCH 166/183] Fix: Write hash for live photo --- pkg/model/remote.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/model/remote.go b/pkg/model/remote.go index 75ff88c43..d8d6f3fce 100644 --- a/pkg/model/remote.go +++ b/pkg/model/remote.go @@ -87,6 +87,14 @@ func (r *RemoteFile) IsLivePhoto() bool { func (r *RemoteFile) GetFileHash() *string { value, ok := r.Metadata["hash"] if !ok { + if r.IsLivePhoto() { + imageHash, hasImgHash := r.Metadata["imageHash"] + vidHash, hasVidHash := r.Metadata["videoHash"] + if hasImgHash && hasVidHash { + hash := fmt.Sprintf("%s:%s", imageHash, vidHash) + return &hash + } + } return nil } if str, ok := value.(string); ok { From d3c60b7f9d4d425a7da243fa141611d136c78837 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 30 Oct 2023 10:26:39 +0530 Subject: [PATCH 167/183] Bump version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index bea4a2217..468400ce8 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -11,7 +11,7 @@ import ( "github.com/spf13/cobra" ) -const AppVersion = "0.1.6" +const AppVersion = "0.1.7" var ctrl *pkg.ClICtrl From bb415850cb65cac07d32def892e5d4e5c0720295 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 30 Oct 2023 10:36:34 +0530 Subject: [PATCH 168/183] Fixed CLI db path --- main.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 254b39b9d..ff29b414e 100644 --- a/main.go +++ b/main.go @@ -19,7 +19,15 @@ func main() { if err != nil { log.Fatalf("Could not create cli config path\n%v\n", err) } - db, err := pkg.GetDB(fmt.Sprintf("%sente-cli.db", cliDBPath)) + oldCliPath := fmt.Sprintf("%sente-cli.db", cliDBPath) + newCliPath := fmt.Sprintf("%s/ente-cli.db", cliDBPath) + if _, err := os.Stat(oldCliPath); err == nil { + log.Printf("migrating old cli db from %s to %s\n", oldCliPath, newCliPath) + if err := os.Rename(oldCliPath, newCliPath); err != nil { + log.Fatalf("Could not rename old cli db\n%v\n", err) + } + } + db, err := pkg.GetDB(newCliPath) if secrets.IsRunningInContainer() { cliDBPath = constants.CliDataPath _, err := internal.ValidateDirForWrite(cliDBPath) From 4159974c249dea833aedf4d929ecc20ec0dcffbb Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 1 Nov 2023 10:08:26 +0530 Subject: [PATCH 169/183] Improve logs --- main.go | 15 ++++++++------- pkg/remote_sync.go | 6 +++--- pkg/remote_to_disk_file.go | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/main.go b/main.go index ff29b414e..6424aea40 100644 --- a/main.go +++ b/main.go @@ -16,6 +16,14 @@ import ( func main() { cliDBPath, err := GetCLIConfigPath() + if secrets.IsRunningInContainer() { + cliDBPath = constants.CliDataPath + _, err := internal.ValidateDirForWrite(cliDBPath) + if err != nil { + log.Fatalf("Please mount a volume to %s to persist cli data\n%v\n", cliDBPath, err) + } + } + if err != nil { log.Fatalf("Could not create cli config path\n%v\n", err) } @@ -28,13 +36,6 @@ func main() { } } db, err := pkg.GetDB(newCliPath) - if secrets.IsRunningInContainer() { - cliDBPath = constants.CliDataPath - _, err := internal.ValidateDirForWrite(cliDBPath) - if err != nil { - log.Fatalf("Please mount a volume to %s to persist cli data\n%v\n", cliDBPath, err) - } - } if err != nil { if strings.Contains(err.Error(), "timeout") { diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index 281afc329..a37a4a58c 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -70,13 +70,13 @@ func (c *ClICtrl) fetchRemoteFiles(ctx context.Context) error { break } if isFirstSync { - log.Printf("First sync for album %s\n", album.AlbumName) + log.Printf("Sync files metadata for album %s\n", album.AlbumName) } else { - log.Printf("Syncing album %s\n from %s", album.AlbumName, time.UnixMicro(lastSyncTime)) + log.Printf("Sync files metadata for album %s\n from %s", album.AlbumName, time.UnixMicro(lastSyncTime)) } if !isFirstSync { t := time.UnixMicro(lastSyncTime) - log.Printf("Fetching files for album %s from %v\n", album.AlbumName, t) + log.Printf("Fetching files metadata for album %s from %v\n", album.AlbumName, t) } files, hasMore, err := c.Client.GetFiles(ctx, album.ID, lastSyncTime) if err != nil { diff --git a/pkg/remote_to_disk_file.go b/pkg/remote_to_disk_file.go index 79352e119..fc495e66f 100644 --- a/pkg/remote_to_disk_file.go +++ b/pkg/remote_to_disk_file.go @@ -17,7 +17,7 @@ import ( ) func (c *ClICtrl) syncFiles(ctx context.Context, account model.Account) error { - log.Printf("Starting sync files") + log.Printf("Starting file download") exportRoot := account.ExportDir _, albumIDToMetaMap, err := readFolderMetadata(exportRoot) if err != nil { From 5d4c14a5d5949dfeb6b0a5e5852a9e621ac5f74e Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 1 Dec 2023 20:45:22 +0530 Subject: [PATCH 170/183] Fix DB path --- main.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index 6424aea40..2147afde9 100644 --- a/main.go +++ b/main.go @@ -27,12 +27,14 @@ func main() { if err != nil { log.Fatalf("Could not create cli config path\n%v\n", err) } - oldCliPath := fmt.Sprintf("%sente-cli.db", cliDBPath) newCliPath := fmt.Sprintf("%s/ente-cli.db", cliDBPath) - if _, err := os.Stat(oldCliPath); err == nil { - log.Printf("migrating old cli db from %s to %s\n", oldCliPath, newCliPath) - if err := os.Rename(oldCliPath, newCliPath); err != nil { - log.Fatalf("Could not rename old cli db\n%v\n", err) + if !strings.HasPrefix(cliDBPath, "/") { + oldCliPath := fmt.Sprintf("%sente-cli.db", cliDBPath) + if _, err := os.Stat(oldCliPath); err == nil { + log.Printf("migrating old cli db from %s to %s\n", oldCliPath, newCliPath) + if err := os.Rename(oldCliPath, newCliPath); err != nil { + log.Fatalf("Could not rename old cli db\n%v\n", err) + } } } db, err := pkg.GetDB(newCliPath) From c4f203ac7cf5fb8ee7ed200ada866cdafd81e8e2 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 1 Dec 2023 20:50:47 +0530 Subject: [PATCH 171/183] Continue export on unpacking err --- pkg/remote_to_disk_file.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/remote_to_disk_file.go b/pkg/remote_to_disk_file.go index fc495e66f..a192d48a3 100644 --- a/pkg/remote_to_disk_file.go +++ b/pkg/remote_to_disk_file.go @@ -1,6 +1,7 @@ package pkg import ( + "archive/zip" "context" "encoding/json" "errors" @@ -70,6 +71,9 @@ func (c *ClICtrl) syncFiles(ctx context.Context, account model.Account) error { if err != nil { if errors.Is(err, model.ErrDecryption) { continue + } else if existingEntry.IsLivePhoto() && errors.Is(err, zip.ErrFormat) { + log.Printf(fmt.Sprintf("err processing live photo %s (%d), %s", existingEntry.GetTitle(), existingEntry.ID, err.Error())) + continue } else { return err } From 9529639b963e26c2bc0aed7c2b198ebc49786278 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 2 Dec 2023 00:14:28 +0530 Subject: [PATCH 172/183] Sanitize folder name --- pkg/remote_to_disk_album.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/remote_to_disk_album.go b/pkg/remote_to_disk_album.go index dc1dfeacb..540aee1f7 100644 --- a/pkg/remote_to_disk_album.go +++ b/pkg/remote_to_disk_album.go @@ -49,6 +49,7 @@ func (c *ClICtrl) createLocalFolderForRemoteAlbums(ctx context.Context, account albumFolderName := filepath.Clean(album.AlbumName) // replace : with _ albumFolderName = strings.ReplaceAll(albumFolderName, ":", "_") + albumFolderName = strings.ReplaceAll(albumFolderName, "/", "_") albumID := album.ID if _, ok := folderToMetaMap[albumFolderName]; ok { From 7e921dde5ed0c5fbdcad594b1b006af41baa88c1 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 4 Dec 2023 10:20:11 +0530 Subject: [PATCH 173/183] Handle zip unpack err --- pkg/model/errors.go | 1 + pkg/remote_to_disk_file.go | 46 +++++++++++++++++++++++++------------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/pkg/model/errors.go b/pkg/model/errors.go index d8f61d275..a18df3751 100644 --- a/pkg/model/errors.go +++ b/pkg/model/errors.go @@ -6,6 +6,7 @@ import ( ) var ErrDecryption = errors.New("error while decrypting the file") +var ErrLiveZip = errors.New("error: no image or video file found in zip") func ShouldRetrySync(err error) bool { return strings.Contains(err.Error(), "read tcp") || diff --git a/pkg/remote_to_disk_file.go b/pkg/remote_to_disk_file.go index a192d48a3..a980d5aff 100644 --- a/pkg/remote_to_disk_file.go +++ b/pkg/remote_to_disk_file.go @@ -66,6 +66,9 @@ func (c *ClICtrl) syncFiles(ctx context.Context, account model.Account) error { if err != nil { return err } + if !existingEntry.IsLivePhoto() { + continue + } log.Printf("[%d/%d] Sync %s for album %s", i, len(entries), existingEntry.GetTitle(), albumInfo.AlbumName) err = c.downloadEntry(ctx, albumDiskInfo, *existingEntry, entry) if err != nil { @@ -74,14 +77,17 @@ func (c *ClICtrl) syncFiles(ctx context.Context, account model.Account) error { } else if existingEntry.IsLivePhoto() && errors.Is(err, zip.ErrFormat) { log.Printf(fmt.Sprintf("err processing live photo %s (%d), %s", existingEntry.GetTitle(), existingEntry.ID, err.Error())) continue + } else if existingEntry.IsLivePhoto() && errors.Is(err, model.ErrLiveZip) { + continue } else { return err } } } else { - log.Fatalf("remoteFile %d not found in remoteFiles", entry.FileID) + log.Fatalf("File %d not found in remote", entry.FileID) } } + return nil } @@ -137,23 +143,31 @@ func (c *ClICtrl) downloadEntry(ctx context.Context, if err != nil { return err } - imageExtn := filepath.Ext(imagePath) - videoExtn := filepath.Ext(videoPath) - imageFileName := diskInfo.GenerateUniqueFileName(baseFileName, imageExtn) - videoFileName := diskInfo.GenerateUniqueFileName(baseFileName, videoExtn) - imageFilePath := filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, imageFileName) - videoFilePath := filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, videoFileName) - // move the decrypt file to filePath - err = Move(imagePath, imageFilePath) - if err != nil { - return err + if imagePath == "" && videoPath == "" { + log.Printf("imagePath %s, videoPath %s", imagePath, videoPath) + return model.ErrLiveZip } - err = Move(videoPath, videoFilePath) - if err != nil { - return err + if imagePath != "" { + imageExtn := filepath.Ext(imagePath) + imageFileName := diskInfo.GenerateUniqueFileName(baseFileName, imageExtn) + imageFilePath := filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, imageFileName) + moveErr := Move(imagePath, imageFilePath) + if moveErr != nil { + return moveErr + } + fileDiskMetadata.AddFileName(imageFileName) + } + if videoPath == "" { + videoExtn := filepath.Ext(videoPath) + videoFileName := diskInfo.GenerateUniqueFileName(baseFileName, videoExtn) + videoFilePath := filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, videoFileName) + // move the decrypt file to filePath + moveErr := Move(videoPath, videoFilePath) + if moveErr != nil { + return moveErr + } + fileDiskMetadata.AddFileName(videoFileName) } - fileDiskMetadata.AddFileName(imageFileName) - fileDiskMetadata.AddFileName(videoFileName) } else { fileName := diskInfo.GenerateUniqueFileName(baseFileName, extension) filePath := filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, fileName) From 70e9e47f8e2d10cc3d4d060e6f572f2963231b3c Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 4 Dec 2023 10:22:50 +0530 Subject: [PATCH 174/183] Remove check for live photo --- pkg/remote_to_disk_file.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/remote_to_disk_file.go b/pkg/remote_to_disk_file.go index a980d5aff..413182566 100644 --- a/pkg/remote_to_disk_file.go +++ b/pkg/remote_to_disk_file.go @@ -66,9 +66,6 @@ func (c *ClICtrl) syncFiles(ctx context.Context, account model.Account) error { if err != nil { return err } - if !existingEntry.IsLivePhoto() { - continue - } log.Printf("[%d/%d] Sync %s for album %s", i, len(entries), existingEntry.GetTitle(), albumInfo.AlbumName) err = c.downloadEntry(ctx, albumDiskInfo, *existingEntry, entry) if err != nil { From a42982b76ff550b2b45e7d2da7e1310aaa3a86ed Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 4 Dec 2023 10:23:14 +0530 Subject: [PATCH 175/183] Bump version 0.1.8 --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 468400ce8..153661ccc 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -11,7 +11,7 @@ import ( "github.com/spf13/cobra" ) -const AppVersion = "0.1.7" +const AppVersion = "0.1.8" var ctrl *pkg.ClICtrl From 33994fac5ef997f2a3b2f9a3de88639f4fca6ecc Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 7 Dec 2023 14:11:28 +0530 Subject: [PATCH 176/183] Fix input prompt to handle spaces --- internal/promt.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/promt.go b/internal/promt.go index 5c709ae2e..2e988ac24 100644 --- a/internal/promt.go +++ b/internal/promt.go @@ -1,11 +1,13 @@ package internal import ( + "bufio" "errors" "fmt" "github.com/ente-io/cli/internal/api" "log" "os" + "strings" "golang.org/x/term" ) @@ -22,10 +24,13 @@ func GetSensitiveField(label string) (string, error) { func GetUserInput(label string) (string, error) { fmt.Printf("%s: ", label) var input string - _, err := fmt.Scanln(&input) + reader := bufio.NewReader(os.Stdin) + input, err := reader.ReadString('\n') + //_, err := fmt.Scanln(&input) if err != nil { return "", err } + input = strings.TrimSpace(input) if input == "" { return "", errors.New("input cannot be empty") } From 8e0eaae27739566d45e9ad4ea1405792104e085e Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 7 Dec 2023 14:16:06 +0530 Subject: [PATCH 177/183] Fix: trim spaces from albumFolder name for disk --- pkg/remote_to_disk_album.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/remote_to_disk_album.go b/pkg/remote_to_disk_album.go index 540aee1f7..7b0b64f97 100644 --- a/pkg/remote_to_disk_album.go +++ b/pkg/remote_to_disk_album.go @@ -50,6 +50,8 @@ func (c *ClICtrl) createLocalFolderForRemoteAlbums(ctx context.Context, account // replace : with _ albumFolderName = strings.ReplaceAll(albumFolderName, ":", "_") albumFolderName = strings.ReplaceAll(albumFolderName, "/", "_") + albumFolderName = strings.TrimSpace(albumFolderName) + albumID := album.ID if _, ok := folderToMetaMap[albumFolderName]; ok { From e8558b17f5727d32de278635c0f5c81dc13d7394 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 7 Dec 2023 14:16:31 +0530 Subject: [PATCH 178/183] Bump version 0.1.9 --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 153661ccc..ee75215f8 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -11,7 +11,7 @@ import ( "github.com/spf13/cobra" ) -const AppVersion = "0.1.8" +const AppVersion = "0.1.9" var ctrl *pkg.ClICtrl From cc0ea567988414c811b998bcc7cdf65025d3a766 Mon Sep 17 00:00:00 2001 From: Paul Hovey Date: Mon, 25 Dec 2023 14:58:38 -0600 Subject: [PATCH 179/183] attempt at fixing existing filename issue when exporting --- pkg/disk.go | 16 ++++++++++++++++ pkg/disk_test.go | 32 ++++++++++++++++++++++++++++++++ pkg/remote_to_disk_file.go | 16 +++------------- 3 files changed, 51 insertions(+), 13 deletions(-) create mode 100644 pkg/disk_test.go diff --git a/pkg/disk.go b/pkg/disk.go index 586c1f53a..eb4492300 100644 --- a/pkg/disk.go +++ b/pkg/disk.go @@ -74,6 +74,22 @@ func (a *albumDiskInfo) IsMetaFileNamePresent(metaFileName string) bool { return ok } +// GenerateUniqueMetaFileName generates a unique metafile name. +func (a *albumDiskInfo) GenerateUniqueMetaFileName(baseFileName, extension string) string { + potentialDiskFileName := fmt.Sprintf("%s%s.json", baseFileName, extension) + count := 1 + for a.IsMetaFileNamePresent(potentialDiskFileName) { + // separate the file name and extension + fileName := fmt.Sprintf("%s_%d", baseFileName, count) + potentialDiskFileName = fmt.Sprintf("%s%s.json", fileName, extension) + count++ + if !a.IsMetaFileNamePresent(potentialDiskFileName) { + break + } + } + return potentialDiskFileName +} + // GenerateUniqueFileName generates a unique file name. func (a *albumDiskInfo) GenerateUniqueFileName(baseFileName, extension string) string { fileName := fmt.Sprintf("%s%s", baseFileName, extension) diff --git a/pkg/disk_test.go b/pkg/disk_test.go new file mode 100644 index 000000000..4f5262221 --- /dev/null +++ b/pkg/disk_test.go @@ -0,0 +1,32 @@ +package pkg + +import ( + "path/filepath" + "strings" + "testing" +) + +func TestGenerateUniqueFileName(t *testing.T) { + existingFilenames := make(map[string]bool) + testFilename := "FullSizeRender.jpg" // what Apple calls shared files + + existingFilenames[strings.ToLower(testFilename)] = true + + a := &albumDiskInfo{ + FileNames: &existingFilenames, + } + + // this is taken from downloadEntry() + extension := filepath.Ext(testFilename) + baseFileName := strings.TrimSuffix(filepath.Clean(filepath.Base(testFilename)), extension) + + for i := 0; i < 100; i++ { + newFilename := a.GenerateUniqueFileName(baseFileName, extension) + if strings.Contains(newFilename, "_1_2") { + t.Fatalf("Filename contained _1_2") + } else { + // add generated name to existing files + existingFilenames[strings.ToLower(newFilename)] = true + } + } +} diff --git a/pkg/remote_to_disk_file.go b/pkg/remote_to_disk_file.go index 413182566..382751853 100644 --- a/pkg/remote_to_disk_file.go +++ b/pkg/remote_to_disk_file.go @@ -124,17 +124,7 @@ func (c *ClICtrl) downloadEntry(ctx context.Context, // Get the extension extension := filepath.Ext(fileDiskMetadata.Title) baseFileName := strings.TrimSuffix(filepath.Clean(filepath.Base(fileDiskMetadata.Title)), extension) - potentialDiskFileName := fmt.Sprintf("%s%s.json", baseFileName, extension) - count := 1 - for diskInfo.IsMetaFileNamePresent(potentialDiskFileName) { - // separate the file name and extension - baseFileName = fmt.Sprintf("%s_%d", baseFileName, count) - potentialDiskFileName = fmt.Sprintf("%s%s.json", baseFileName, extension) - count++ - if !diskInfo.IsMetaFileNamePresent(potentialDiskFileName) { - break - } - } + diskMetaFileName := diskInfo.GenerateUniqueMetaFileName(baseFileName, extension) if file.IsLivePhoto() { imagePath, videoPath, err := UnpackLive(*decrypt) if err != nil { @@ -176,13 +166,13 @@ func (c *ClICtrl) downloadEntry(ctx context.Context, fileDiskMetadata.AddFileName(fileName) } - fileDiskMetadata.MetaFileName = potentialDiskFileName + fileDiskMetadata.MetaFileName = diskMetaFileName err = diskInfo.AddEntry(fileDiskMetadata) if err != nil { return err } - err = writeJSONToFile(filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, ".meta", potentialDiskFileName), fileDiskMetadata) + err = writeJSONToFile(filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, ".meta", diskMetaFileName), fileDiskMetadata) if err != nil { return err } From f775681162f0f7cee7e3b4ae7c188a8a037794a8 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 5 Jan 2024 11:09:05 +0530 Subject: [PATCH 180/183] Fix bug in handling deleted entry --- pkg/remote_to_disk_file.go | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/pkg/remote_to_disk_file.go b/pkg/remote_to_disk_file.go index 413182566..33090c645 100644 --- a/pkg/remote_to_disk_file.go +++ b/pkg/remote_to_disk_file.go @@ -32,18 +32,17 @@ func (c *ClICtrl) syncFiles(ctx context.Context, account model.Account) error { model.SortAlbumFileEntry(entries) defer utils.TimeTrack(time.Now(), "process_files") var albumDiskInfo *albumDiskInfo - for i, entry := range entries { - if entry.SyncedLocally { + for i, albumFileEntry := range entries { + if albumFileEntry.SyncedLocally { continue } - albumInfo, ok := albumIDToMetaMap[entry.AlbumID] + albumInfo, ok := albumIDToMetaMap[albumFileEntry.AlbumID] if !ok { - log.Printf("Album %d not found in local metadata", entry.AlbumID) + log.Printf("Album %d not found in local metadata", albumFileEntry.AlbumID) continue } - if albumInfo.IsDeleted { - putErr := c.DeleteAlbumEntry(ctx, entry) + putErr := c.DeleteAlbumEntry(ctx, albumFileEntry) if putErr != nil { return putErr } @@ -56,7 +55,7 @@ func (c *ClICtrl) syncFiles(ctx context.Context, account model.Account) error { return err } } - fileBytes, err := c.GetValue(ctx, model.RemoteFiles, []byte(fmt.Sprintf("%d", entry.FileID))) + fileBytes, err := c.GetValue(ctx, model.RemoteFiles, []byte(fmt.Sprintf("%d", albumFileEntry.FileID))) if err != nil { return err } @@ -67,7 +66,7 @@ func (c *ClICtrl) syncFiles(ctx context.Context, account model.Account) error { return err } log.Printf("[%d/%d] Sync %s for album %s", i, len(entries), existingEntry.GetTitle(), albumInfo.AlbumName) - err = c.downloadEntry(ctx, albumDiskInfo, *existingEntry, entry) + err = c.downloadEntry(ctx, albumDiskInfo, *existingEntry, albumFileEntry) if err != nil { if errors.Is(err, model.ErrDecryption) { continue @@ -81,7 +80,15 @@ func (c *ClICtrl) syncFiles(ctx context.Context, account model.Account) error { } } } else { - log.Fatalf("File %d not found in remote", entry.FileID) + // file metadata is missing in the localDB + if albumFileEntry.IsDeleted { + delErr := c.DeleteAlbumEntry(ctx, albumFileEntry) + if delErr != nil { + log.Fatalf("Error deleting album entry %d (deleted: %v) %v", albumFileEntry.FileID, albumFileEntry.IsDeleted, delErr) + } + } else { + log.Fatalf("Failed to find entry in db for file %d (deleted: %v)", albumFileEntry.FileID, albumFileEntry.IsDeleted) + } } } From a71a9f47f16b03181e58564d822d3b3f456daa2b Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 5 Jan 2024 11:09:58 +0530 Subject: [PATCH 181/183] Add support to fetch specific file --- internal/api/api_error.go | 27 +++++++++++++++++++++++++++ internal/api/collection.go | 20 ++++++++++++++++++++ internal/api/login_type.go | 11 ----------- 3 files changed, 47 insertions(+), 11 deletions(-) create mode 100644 internal/api/api_error.go diff --git a/internal/api/api_error.go b/internal/api/api_error.go new file mode 100644 index 000000000..0b59eb180 --- /dev/null +++ b/internal/api/api_error.go @@ -0,0 +1,27 @@ +package api + +import ( + "fmt" + "strings" +) + +type ApiError struct { + Message string + StatusCode int +} + +func (e *ApiError) Error() string { + return fmt.Sprintf("status %d with err: %s", e.StatusCode, e.Message) +} + +func IsApiError(err error) bool { + _, ok := err.(*ApiError) + return ok +} + +func IsFileNotInAlbumError(err error) bool { + if apiErr, ok := err.(*ApiError); ok { + return strings.Contains(apiErr.Message, "FILE_NOT_FOUND_IN_ALBUM") + } + return false +} diff --git a/internal/api/collection.go b/internal/api/collection.go index 6af702381..f8c4c24d4 100644 --- a/internal/api/collection.go +++ b/internal/api/collection.go @@ -42,3 +42,23 @@ func (c *Client) GetFiles(ctx context.Context, collectionID, sinceTime int64) ([ } return res.Files, res.HasMore, err } + +// GetFile .. +func (c *Client) GetFile(ctx context.Context, collectionID, fileID int64) (*File, error) { + var res struct { + File File `json:"file"` + } + r, err := c.restClient.R(). + SetContext(ctx). + SetQueryParam("collectionID", strconv.FormatInt(collectionID, 10)). + SetQueryParam("fileID", strconv.FormatInt(fileID, 10)). + SetResult(&res). + Get("/collections/file") + if r.IsError() { + return nil, &ApiError{ + StatusCode: r.StatusCode(), + Message: r.String(), + } + } + return &res.File, err +} diff --git a/internal/api/login_type.go b/internal/api/login_type.go index 9c60d4eb1..50149aed7 100644 --- a/internal/api/login_type.go +++ b/internal/api/login_type.go @@ -1,8 +1,6 @@ package api import ( - "fmt" - "github.com/google/uuid" ) @@ -15,15 +13,6 @@ type SRPAttributes struct { IsEmailMFAEnabled bool `json:"isEmailMFAEnabled" binding:"required"` } -type ApiError struct { - Message string - StatusCode int -} - -func (e *ApiError) Error() string { - return fmt.Sprintf("status %d with err: %s", e.StatusCode, e.Message) -} - type CreateSRPSessionResponse struct { SessionID uuid.UUID `json:"sessionID" binding:"required"` SRPB string `json:"srpB" binding:"required"` From 8485e80f42cdb523076ca3a6a206660fb55d33c8 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 5 Jan 2024 11:19:46 +0530 Subject: [PATCH 182/183] Bump version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index ee75215f8..6ed7c380a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -11,7 +11,7 @@ import ( "github.com/spf13/cobra" ) -const AppVersion = "0.1.9" +const AppVersion = "0.1.10" var ctrl *pkg.ClICtrl From 39ef006532154efeac90c411a7ea4c50c8f4cc8d Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 1 Mar 2024 12:39:01 +0530 Subject: [PATCH 183/183] Move into cli/ --- {.github => cli/.github}/workflows/release.yml | 0 .gitignore => cli/.gitignore | 0 .goreleaser.yaml => cli/.goreleaser.yaml | 0 Dockerfile => cli/Dockerfile | 0 Dockerfile-x86 => cli/Dockerfile-x86 | 0 README.md => cli/README.md | 0 {cmd => cli/cmd}/LICENSE | 0 {cmd => cli/cmd}/account.go | 0 {cmd => cli/cmd}/config.go | 0 {cmd => cli/cmd}/export.go | 0 {cmd => cli/cmd}/root.go | 0 {cmd => cli/cmd}/version.go | 0 config.yaml => cli/config.yaml | 0 docker-compose.yml => cli/docker-compose.yml | 0 go.mod => cli/go.mod | 0 go.sum => cli/go.sum | 0 {internal => cli/internal}/api/api_error.go | 0 {internal => cli/internal}/api/client.go | 0 {internal => cli/internal}/api/collection.go | 0 {internal => cli/internal}/api/collection_type.go | 0 {internal => cli/internal}/api/enums.go | 0 {internal => cli/internal}/api/file_type.go | 0 {internal => cli/internal}/api/files.go | 0 {internal => cli/internal}/api/log.go | 0 {internal => cli/internal}/api/login.go | 0 {internal => cli/internal}/api/login_type.go | 0 {internal => cli/internal}/crypto/crypto.go | 0 {internal => cli/internal}/crypto/crypto_libsodium.go | 0 {internal => cli/internal}/crypto/crypto_test.go | 0 {internal => cli/internal}/crypto/stream.go | 0 {internal => cli/internal}/crypto/utils.go | 0 {internal => cli/internal}/promt.go | 0 main.go => cli/main.go | 0 {pkg => cli/pkg}/account.go | 0 {pkg => cli/pkg}/bolt_store.go | 0 {pkg => cli/pkg}/cli.go | 0 {pkg => cli/pkg}/disk.go | 0 {pkg => cli/pkg}/disk_test.go | 0 {pkg => cli/pkg}/download.go | 0 {pkg => cli/pkg}/mapper/photo.go | 0 {pkg => cli/pkg}/model/account.go | 0 {pkg => cli/pkg}/model/constants.go | 0 {pkg => cli/pkg}/model/enc_string.go | 0 {pkg => cli/pkg}/model/enc_string_test.go | 0 {pkg => cli/pkg}/model/errors.go | 0 {pkg => cli/pkg}/model/export/location.go | 0 {pkg => cli/pkg}/model/export/metadata.go | 0 {pkg => cli/pkg}/model/remote.go | 0 {pkg => cli/pkg}/remote_sync.go | 0 {pkg => cli/pkg}/remote_to_disk_album.go | 0 {pkg => cli/pkg}/remote_to_disk_file.go | 0 {pkg => cli/pkg}/secrets/key_holder.go | 0 {pkg => cli/pkg}/secrets/secret.go | 0 {pkg => cli/pkg}/sign_in.go | 0 {pkg => cli/pkg}/store.go | 0 {pkg => cli/pkg}/sync.go | 0 release.sh => cli/release.sh | 0 {utils => cli/utils}/constants/constants.go | 0 {utils => cli/utils}/encoding/encoding.go | 0 {utils => cli/utils}/time.go | 0 60 files changed, 0 insertions(+), 0 deletions(-) rename {.github => cli/.github}/workflows/release.yml (100%) rename .gitignore => cli/.gitignore (100%) rename .goreleaser.yaml => cli/.goreleaser.yaml (100%) rename Dockerfile => cli/Dockerfile (100%) rename Dockerfile-x86 => cli/Dockerfile-x86 (100%) rename README.md => cli/README.md (100%) rename {cmd => cli/cmd}/LICENSE (100%) rename {cmd => cli/cmd}/account.go (100%) rename {cmd => cli/cmd}/config.go (100%) rename {cmd => cli/cmd}/export.go (100%) rename {cmd => cli/cmd}/root.go (100%) rename {cmd => cli/cmd}/version.go (100%) rename config.yaml => cli/config.yaml (100%) rename docker-compose.yml => cli/docker-compose.yml (100%) rename go.mod => cli/go.mod (100%) rename go.sum => cli/go.sum (100%) rename {internal => cli/internal}/api/api_error.go (100%) rename {internal => cli/internal}/api/client.go (100%) rename {internal => cli/internal}/api/collection.go (100%) rename {internal => cli/internal}/api/collection_type.go (100%) rename {internal => cli/internal}/api/enums.go (100%) rename {internal => cli/internal}/api/file_type.go (100%) rename {internal => cli/internal}/api/files.go (100%) rename {internal => cli/internal}/api/log.go (100%) rename {internal => cli/internal}/api/login.go (100%) rename {internal => cli/internal}/api/login_type.go (100%) rename {internal => cli/internal}/crypto/crypto.go (100%) rename {internal => cli/internal}/crypto/crypto_libsodium.go (100%) rename {internal => cli/internal}/crypto/crypto_test.go (100%) rename {internal => cli/internal}/crypto/stream.go (100%) rename {internal => cli/internal}/crypto/utils.go (100%) rename {internal => cli/internal}/promt.go (100%) rename main.go => cli/main.go (100%) rename {pkg => cli/pkg}/account.go (100%) rename {pkg => cli/pkg}/bolt_store.go (100%) rename {pkg => cli/pkg}/cli.go (100%) rename {pkg => cli/pkg}/disk.go (100%) rename {pkg => cli/pkg}/disk_test.go (100%) rename {pkg => cli/pkg}/download.go (100%) rename {pkg => cli/pkg}/mapper/photo.go (100%) rename {pkg => cli/pkg}/model/account.go (100%) rename {pkg => cli/pkg}/model/constants.go (100%) rename {pkg => cli/pkg}/model/enc_string.go (100%) rename {pkg => cli/pkg}/model/enc_string_test.go (100%) rename {pkg => cli/pkg}/model/errors.go (100%) rename {pkg => cli/pkg}/model/export/location.go (100%) rename {pkg => cli/pkg}/model/export/metadata.go (100%) rename {pkg => cli/pkg}/model/remote.go (100%) rename {pkg => cli/pkg}/remote_sync.go (100%) rename {pkg => cli/pkg}/remote_to_disk_album.go (100%) rename {pkg => cli/pkg}/remote_to_disk_file.go (100%) rename {pkg => cli/pkg}/secrets/key_holder.go (100%) rename {pkg => cli/pkg}/secrets/secret.go (100%) rename {pkg => cli/pkg}/sign_in.go (100%) rename {pkg => cli/pkg}/store.go (100%) rename {pkg => cli/pkg}/sync.go (100%) rename release.sh => cli/release.sh (100%) rename {utils => cli/utils}/constants/constants.go (100%) rename {utils => cli/utils}/encoding/encoding.go (100%) rename {utils => cli/utils}/time.go (100%) diff --git a/.github/workflows/release.yml b/cli/.github/workflows/release.yml similarity index 100% rename from .github/workflows/release.yml rename to cli/.github/workflows/release.yml diff --git a/.gitignore b/cli/.gitignore similarity index 100% rename from .gitignore rename to cli/.gitignore diff --git a/.goreleaser.yaml b/cli/.goreleaser.yaml similarity index 100% rename from .goreleaser.yaml rename to cli/.goreleaser.yaml diff --git a/Dockerfile b/cli/Dockerfile similarity index 100% rename from Dockerfile rename to cli/Dockerfile diff --git a/Dockerfile-x86 b/cli/Dockerfile-x86 similarity index 100% rename from Dockerfile-x86 rename to cli/Dockerfile-x86 diff --git a/README.md b/cli/README.md similarity index 100% rename from README.md rename to cli/README.md diff --git a/cmd/LICENSE b/cli/cmd/LICENSE similarity index 100% rename from cmd/LICENSE rename to cli/cmd/LICENSE diff --git a/cmd/account.go b/cli/cmd/account.go similarity index 100% rename from cmd/account.go rename to cli/cmd/account.go diff --git a/cmd/config.go b/cli/cmd/config.go similarity index 100% rename from cmd/config.go rename to cli/cmd/config.go diff --git a/cmd/export.go b/cli/cmd/export.go similarity index 100% rename from cmd/export.go rename to cli/cmd/export.go diff --git a/cmd/root.go b/cli/cmd/root.go similarity index 100% rename from cmd/root.go rename to cli/cmd/root.go diff --git a/cmd/version.go b/cli/cmd/version.go similarity index 100% rename from cmd/version.go rename to cli/cmd/version.go diff --git a/config.yaml b/cli/config.yaml similarity index 100% rename from config.yaml rename to cli/config.yaml diff --git a/docker-compose.yml b/cli/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to cli/docker-compose.yml diff --git a/go.mod b/cli/go.mod similarity index 100% rename from go.mod rename to cli/go.mod diff --git a/go.sum b/cli/go.sum similarity index 100% rename from go.sum rename to cli/go.sum diff --git a/internal/api/api_error.go b/cli/internal/api/api_error.go similarity index 100% rename from internal/api/api_error.go rename to cli/internal/api/api_error.go diff --git a/internal/api/client.go b/cli/internal/api/client.go similarity index 100% rename from internal/api/client.go rename to cli/internal/api/client.go diff --git a/internal/api/collection.go b/cli/internal/api/collection.go similarity index 100% rename from internal/api/collection.go rename to cli/internal/api/collection.go diff --git a/internal/api/collection_type.go b/cli/internal/api/collection_type.go similarity index 100% rename from internal/api/collection_type.go rename to cli/internal/api/collection_type.go diff --git a/internal/api/enums.go b/cli/internal/api/enums.go similarity index 100% rename from internal/api/enums.go rename to cli/internal/api/enums.go diff --git a/internal/api/file_type.go b/cli/internal/api/file_type.go similarity index 100% rename from internal/api/file_type.go rename to cli/internal/api/file_type.go diff --git a/internal/api/files.go b/cli/internal/api/files.go similarity index 100% rename from internal/api/files.go rename to cli/internal/api/files.go diff --git a/internal/api/log.go b/cli/internal/api/log.go similarity index 100% rename from internal/api/log.go rename to cli/internal/api/log.go diff --git a/internal/api/login.go b/cli/internal/api/login.go similarity index 100% rename from internal/api/login.go rename to cli/internal/api/login.go diff --git a/internal/api/login_type.go b/cli/internal/api/login_type.go similarity index 100% rename from internal/api/login_type.go rename to cli/internal/api/login_type.go diff --git a/internal/crypto/crypto.go b/cli/internal/crypto/crypto.go similarity index 100% rename from internal/crypto/crypto.go rename to cli/internal/crypto/crypto.go diff --git a/internal/crypto/crypto_libsodium.go b/cli/internal/crypto/crypto_libsodium.go similarity index 100% rename from internal/crypto/crypto_libsodium.go rename to cli/internal/crypto/crypto_libsodium.go diff --git a/internal/crypto/crypto_test.go b/cli/internal/crypto/crypto_test.go similarity index 100% rename from internal/crypto/crypto_test.go rename to cli/internal/crypto/crypto_test.go diff --git a/internal/crypto/stream.go b/cli/internal/crypto/stream.go similarity index 100% rename from internal/crypto/stream.go rename to cli/internal/crypto/stream.go diff --git a/internal/crypto/utils.go b/cli/internal/crypto/utils.go similarity index 100% rename from internal/crypto/utils.go rename to cli/internal/crypto/utils.go diff --git a/internal/promt.go b/cli/internal/promt.go similarity index 100% rename from internal/promt.go rename to cli/internal/promt.go diff --git a/main.go b/cli/main.go similarity index 100% rename from main.go rename to cli/main.go diff --git a/pkg/account.go b/cli/pkg/account.go similarity index 100% rename from pkg/account.go rename to cli/pkg/account.go diff --git a/pkg/bolt_store.go b/cli/pkg/bolt_store.go similarity index 100% rename from pkg/bolt_store.go rename to cli/pkg/bolt_store.go diff --git a/pkg/cli.go b/cli/pkg/cli.go similarity index 100% rename from pkg/cli.go rename to cli/pkg/cli.go diff --git a/pkg/disk.go b/cli/pkg/disk.go similarity index 100% rename from pkg/disk.go rename to cli/pkg/disk.go diff --git a/pkg/disk_test.go b/cli/pkg/disk_test.go similarity index 100% rename from pkg/disk_test.go rename to cli/pkg/disk_test.go diff --git a/pkg/download.go b/cli/pkg/download.go similarity index 100% rename from pkg/download.go rename to cli/pkg/download.go diff --git a/pkg/mapper/photo.go b/cli/pkg/mapper/photo.go similarity index 100% rename from pkg/mapper/photo.go rename to cli/pkg/mapper/photo.go diff --git a/pkg/model/account.go b/cli/pkg/model/account.go similarity index 100% rename from pkg/model/account.go rename to cli/pkg/model/account.go diff --git a/pkg/model/constants.go b/cli/pkg/model/constants.go similarity index 100% rename from pkg/model/constants.go rename to cli/pkg/model/constants.go diff --git a/pkg/model/enc_string.go b/cli/pkg/model/enc_string.go similarity index 100% rename from pkg/model/enc_string.go rename to cli/pkg/model/enc_string.go diff --git a/pkg/model/enc_string_test.go b/cli/pkg/model/enc_string_test.go similarity index 100% rename from pkg/model/enc_string_test.go rename to cli/pkg/model/enc_string_test.go diff --git a/pkg/model/errors.go b/cli/pkg/model/errors.go similarity index 100% rename from pkg/model/errors.go rename to cli/pkg/model/errors.go diff --git a/pkg/model/export/location.go b/cli/pkg/model/export/location.go similarity index 100% rename from pkg/model/export/location.go rename to cli/pkg/model/export/location.go diff --git a/pkg/model/export/metadata.go b/cli/pkg/model/export/metadata.go similarity index 100% rename from pkg/model/export/metadata.go rename to cli/pkg/model/export/metadata.go diff --git a/pkg/model/remote.go b/cli/pkg/model/remote.go similarity index 100% rename from pkg/model/remote.go rename to cli/pkg/model/remote.go diff --git a/pkg/remote_sync.go b/cli/pkg/remote_sync.go similarity index 100% rename from pkg/remote_sync.go rename to cli/pkg/remote_sync.go diff --git a/pkg/remote_to_disk_album.go b/cli/pkg/remote_to_disk_album.go similarity index 100% rename from pkg/remote_to_disk_album.go rename to cli/pkg/remote_to_disk_album.go diff --git a/pkg/remote_to_disk_file.go b/cli/pkg/remote_to_disk_file.go similarity index 100% rename from pkg/remote_to_disk_file.go rename to cli/pkg/remote_to_disk_file.go diff --git a/pkg/secrets/key_holder.go b/cli/pkg/secrets/key_holder.go similarity index 100% rename from pkg/secrets/key_holder.go rename to cli/pkg/secrets/key_holder.go diff --git a/pkg/secrets/secret.go b/cli/pkg/secrets/secret.go similarity index 100% rename from pkg/secrets/secret.go rename to cli/pkg/secrets/secret.go diff --git a/pkg/sign_in.go b/cli/pkg/sign_in.go similarity index 100% rename from pkg/sign_in.go rename to cli/pkg/sign_in.go diff --git a/pkg/store.go b/cli/pkg/store.go similarity index 100% rename from pkg/store.go rename to cli/pkg/store.go diff --git a/pkg/sync.go b/cli/pkg/sync.go similarity index 100% rename from pkg/sync.go rename to cli/pkg/sync.go diff --git a/release.sh b/cli/release.sh similarity index 100% rename from release.sh rename to cli/release.sh diff --git a/utils/constants/constants.go b/cli/utils/constants/constants.go similarity index 100% rename from utils/constants/constants.go rename to cli/utils/constants/constants.go diff --git a/utils/encoding/encoding.go b/cli/utils/encoding/encoding.go similarity index 100% rename from utils/encoding/encoding.go rename to cli/utils/encoding/encoding.go diff --git a/utils/time.go b/cli/utils/time.go similarity index 100% rename from utils/time.go rename to cli/utils/time.go