replace sqlite by sled for id mapping storing

This commit is contained in:
Clément DOUIN 2024-02-24 09:37:55 +01:00
parent 0e35a0cd64
commit e945c4b8e2
No known key found for this signature in database
GPG key ID: 353E4A18EE0FAB72
6 changed files with 156 additions and 183 deletions

View file

@ -9,12 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added ### Added
- Added account check-up command.
- Added wizard warning about google passwords [#41]. - Added wizard warning about google passwords [#41].
### Changed ### Changed
- Removed account configurations flatten level in order to improve diagnostic errors, due to a [bug](https://github.com/toml-rs/toml/issues/589#issuecomment-1872345017) in clap. **This means that accounts need to be prefixed by `accounts`: `[my-account]` becomes `[accounts.my-account]`**. It also opens doors for interface-specific configurations. - Removed account configurations flatten level in order to improve diagnostic errors, due to a [bug](https://github.com/toml-rs/toml/issues/589#issuecomment-1872345017) in clap. **This means that accounts need to be prefixed by `accounts`: `[my-account]` becomes `[accounts.my-account]`**. It also opens doors for interface-specific configurations.
- Rolled back cargo feature additions from the previous release. It was a mistake: the amount of features was too big, the code (both CLI and lib) was too hard to maintain. Cargo features kept: `imap`, `maildir`, `notmuch`, `smtp`, `sendmail`, `account-sync`, `account-discovery`, `pgp-gpg`, `pgp-commands` and `pgp-native`. - Rolled back cargo feature additions from the previous release. It was a mistake: the amount of features was too big, the code (both CLI and lib) was too hard to maintain. Cargo features kept: `imap`, `maildir`, `notmuch`, `smtp`, `sendmail`, `account-sync`, `account-discovery`, `pgp-gpg`, `pgp-commands` and `pgp-native`.
- Replaced id mapping database `SQLite` by `sled`, a pure key-val store written in Rust to improve portability of the tool. Therefore, id aliases are reset.
- Improved pre and post edit choices interaction [#58]. - Improved pre and post edit choices interaction [#58].
- Improved account synchronization performances, making it 50% faster than `mbsync` and 370% faster than `OfflineIMAP`. - Improved account synchronization performances, making it 50% faster than `mbsync` and 370% faster than `OfflineIMAP`.
- Changed `envelope.watch.{event}.{hook}`: hooks can now be cumulated. For example it is possible to send a system notification and execute a shell command when receiving a new envelope: - Changed `envelope.watch.{event}.{hook}`: hooks can now be cumulated. For example it is possible to send a system notification and execute a shell command when receiving a new envelope:

140
Cargo.lock generated
View file

@ -1217,8 +1217,7 @@ dependencies = [
[[package]] [[package]]
name = "email-lib" name = "email-lib"
version = "0.22.0" version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "git+https://git.sr.ht/~soywod/pimalaya#9cdfca0f6729de2ebc0309883b8c78d0bbdbf31e"
checksum = "37e3535a72128056ee823c40edcf682d3d79c88fe6d86def04f5ae36ce229075"
dependencies = [ dependencies = [
"advisory-lock", "advisory-lock",
"anyhow", "anyhow",
@ -1408,18 +1407,6 @@ dependencies = [
"pin-project-lite", "pin-project-lite",
] ]
[[package]]
name = "fallible-iterator"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
[[package]]
name = "fallible-streaming-iterator"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]] [[package]]
name = "fastrand" name = "fastrand"
version = "1.9.0" version = "1.9.0"
@ -1459,7 +1446,7 @@ checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"libc", "libc",
"redox_syscall", "redox_syscall 0.4.1",
"windows-sys 0.52.0", "windows-sys 0.52.0",
] ]
@ -1515,6 +1502,16 @@ dependencies = [
"syn 1.0.109", "syn 1.0.109",
] ]
[[package]]
name = "fs2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
dependencies = [
"libc",
"winapi",
]
[[package]] [[package]]
name = "futures" name = "futures"
version = "0.3.29" version = "0.3.29"
@ -1632,6 +1629,15 @@ dependencies = [
"slab", "slab",
] ]
[[package]]
name = "fxhash"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
dependencies = [
"byteorder",
]
[[package]] [[package]]
name = "generic-array" name = "generic-array"
version = "0.14.7" version = "0.14.7"
@ -1753,15 +1759,6 @@ dependencies = [
"allocator-api2", "allocator-api2",
] ]
[[package]]
name = "hashlink"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7"
dependencies = [
"hashbrown",
]
[[package]] [[package]]
name = "heck" name = "heck"
version = "0.4.1" version = "0.4.1"
@ -1829,7 +1826,7 @@ dependencies = [
"ipconfig", "ipconfig",
"lru-cache", "lru-cache",
"once_cell", "once_cell",
"parking_lot", "parking_lot 0.12.1",
"rand", "rand",
"resolv-conf", "resolv-conf",
"rustls 0.21.10", "rustls 0.21.10",
@ -1867,11 +1864,11 @@ dependencies = [
"oauth-lib", "oauth-lib",
"once_cell", "once_cell",
"process-lib", "process-lib",
"rusqlite",
"secret-lib", "secret-lib",
"serde", "serde",
"serde_json", "serde_json",
"shellexpand-utils", "shellexpand-utils",
"sled",
"tempfile", "tempfile",
"termcolor", "termcolor",
"terminal_size", "terminal_size",
@ -2287,18 +2284,7 @@ checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8"
dependencies = [ dependencies = [
"bitflags 2.4.1", "bitflags 2.4.1",
"libc", "libc",
"redox_syscall", "redox_syscall 0.4.1",
]
[[package]]
name = "libsqlite3-sys"
version = "0.26.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326"
dependencies = [
"cc",
"pkg-config",
"vcpkg",
] ]
[[package]] [[package]]
@ -2379,7 +2365,7 @@ dependencies = [
"lru-cache", "lru-cache",
"mail-builder", "mail-builder",
"mail-parser", "mail-parser",
"parking_lot", "parking_lot 0.12.1",
"quick-xml", "quick-xml",
"ring 0.17.7", "ring 0.17.7",
"rustls-pemfile", "rustls-pemfile",
@ -2930,6 +2916,17 @@ version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae"
[[package]]
name = "parking_lot"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
dependencies = [
"instant",
"lock_api",
"parking_lot_core 0.8.6",
]
[[package]] [[package]]
name = "parking_lot" name = "parking_lot"
version = "0.12.1" version = "0.12.1"
@ -2937,7 +2934,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
dependencies = [ dependencies = [
"lock_api", "lock_api",
"parking_lot_core", "parking_lot_core 0.9.9",
]
[[package]]
name = "parking_lot_core"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
dependencies = [
"cfg-if",
"instant",
"libc",
"redox_syscall 0.2.16",
"smallvec",
"winapi",
] ]
[[package]] [[package]]
@ -2948,7 +2959,7 @@ checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"libc", "libc",
"redox_syscall", "redox_syscall 0.4.1",
"smallvec", "smallvec",
"windows-targets 0.48.5", "windows-targets 0.48.5",
] ]
@ -3332,6 +3343,15 @@ dependencies = [
"crossbeam-utils", "crossbeam-utils",
] ]
[[package]]
name = "redox_syscall"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
dependencies = [
"bitflags 1.3.2",
]
[[package]] [[package]]
name = "redox_syscall" name = "redox_syscall"
version = "0.4.1" version = "0.4.1"
@ -3522,20 +3542,6 @@ dependencies = [
"zeroize", "zeroize",
] ]
[[package]]
name = "rusqlite"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2"
dependencies = [
"bitflags 2.4.1",
"fallible-iterator",
"fallible-streaming-iterator",
"hashlink",
"libsqlite3-sys",
"smallvec",
]
[[package]] [[package]]
name = "rustc-demangle" name = "rustc-demangle"
version = "0.1.23" version = "0.1.23"
@ -3957,6 +3963,22 @@ dependencies = [
"autocfg", "autocfg",
] ]
[[package]]
name = "sled"
version = "0.34.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935"
dependencies = [
"crc32fast",
"crossbeam-epoch",
"crossbeam-utils",
"fs2",
"fxhash",
"libc",
"log",
"parking_lot 0.11.2",
]
[[package]] [[package]]
name = "smallvec" name = "smallvec"
version = "1.11.2" version = "1.11.2"
@ -4128,7 +4150,7 @@ checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"fastrand 2.0.1", "fastrand 2.0.1",
"redox_syscall", "redox_syscall 0.4.1",
"rustix 0.38.28", "rustix 0.38.28",
"windows-sys 0.48.0", "windows-sys 0.48.0",
] ]
@ -4495,12 +4517,6 @@ dependencies = [
"getrandom", "getrandom",
] ]
[[package]]
name = "vcpkg"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]] [[package]]
name = "version-compare" name = "version-compare"
version = "0.1.1" version = "0.1.1"

View file

@ -78,6 +78,7 @@ secret-lib = "=0.3.3"
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0" serde_json = "1.0"
shellexpand-utils = "=0.2.0" shellexpand-utils = "=0.2.0"
sled = "=0.34.7"
termcolor = "1.1" termcolor = "1.1"
terminal_size = "0.1" terminal_size = "0.1"
tokio = { version = "1.23", default-features = false, features = ["macros", "rt-multi-thread"] } tokio = { version = "1.23", default-features = false, features = ["macros", "rt-multi-thread"] }
@ -87,16 +88,8 @@ unicode-width = "0.1"
url = "2.2" url = "2.2"
uuid = { version = "0.8", features = ["v4"] } uuid = { version = "0.8", features = ["v4"] }
[target.'cfg(target_env = "musl")'.dependencies.rusqlite]
version = "0.29"
features = []
[target.'cfg(not(target_env = "musl"))'.dependencies.rusqlite]
version = "0.29"
features = ["bundled"]
[target.'cfg(not(windows))'.dependencies.coredump] [target.'cfg(not(windows))'.dependencies.coredump]
version = "0.1" version = "0.1"
# [patch.crates-io] [patch.crates-io]
# email-lib = { git = "https://git.sr.ht/~soywod/pimalaya" } email-lib = { git = "https://git.sr.ht/~soywod/pimalaya" }

View file

@ -96,8 +96,6 @@
linux = mkPackage' null { }; linux = mkPackage' null { };
linux-musl = mkPackage' "x86_64-unknown-linux-musl" (with pkgs.pkgsStatic; { linux-musl = mkPackage' "x86_64-unknown-linux-musl" (with pkgs.pkgsStatic; {
CARGO_BUILD_RUSTFLAGS = "-C target-feature=+crt-static"; CARGO_BUILD_RUSTFLAGS = "-C target-feature=+crt-static";
SQLITE3_STATIC = 1;
SQLITE3_LIB_DIR = "${sqlite.out}/lib";
hardeningDisable = [ "all" ]; hardeningDisable = [ "all" ];
}); });
macos = mkPackage' null (with pkgs.darwin.apple_sdk.frameworks; { macos = mkPackage' null (with pkgs.darwin.apple_sdk.frameworks; {

View file

@ -619,32 +619,20 @@ impl Backend {
match backend_kind { match backend_kind {
#[cfg(feature = "maildir")] #[cfg(feature = "maildir")]
Some(BackendKind::Maildir) => { Some(BackendKind::Maildir) => {
if let Some(mdir_config) = &self.toml_account_config.maildir { if let Some(_) = &self.toml_account_config.maildir {
id_mapper = IdMapper::new( id_mapper = IdMapper::new(&self.backend.account_config, folder)?;
&self.backend.account_config,
folder,
mdir_config.root_dir.clone(),
)?;
} }
} }
#[cfg(feature = "account-sync")] #[cfg(feature = "account-sync")]
Some(BackendKind::MaildirForSync) => { Some(BackendKind::MaildirForSync) => {
id_mapper = IdMapper::new( id_mapper = IdMapper::new(&self.backend.account_config, folder)?;
&self.backend.account_config,
folder,
self.backend.account_config.get_sync_dir()?,
)?;
} }
#[cfg(feature = "notmuch")] #[cfg(feature = "notmuch")]
Some(BackendKind::Notmuch) => { Some(BackendKind::Notmuch) => {
if let Some(notmuch_config) = &self.toml_account_config.notmuch { if let Some(_) = &self.toml_account_config.notmuch {
id_mapper = IdMapper::new( id_mapper = IdMapper::new(&self.backend.account_config, folder)?;
&self.backend.account_config,
folder,
notmuch_config.get_maildir_path()?,
)?;
} }
} }
_ => (), _ => (),

160
src/cache/mod.rs vendored
View file

@ -2,61 +2,34 @@ pub mod arg;
pub mod args; pub mod args;
use anyhow::{anyhow, Context, Result}; use anyhow::{anyhow, Context, Result};
use dirs::data_dir;
use email::account::config::AccountConfig; use email::account::config::AccountConfig;
use log::{debug, trace}; use log::debug;
use std::path::{Path, PathBuf}; use sled::{Config, Db};
use std::collections::HashSet;
const ID_MAPPER_DB_FILE_NAME: &str = ".id-mapper.sqlite";
#[derive(Debug)] #[derive(Debug)]
pub enum IdMapper { pub enum IdMapper {
Dummy, Dummy,
Mapper(String, rusqlite::Connection), Mapper(Db),
} }
impl IdMapper { impl IdMapper {
pub fn find_closest_db_path(dir: impl AsRef<Path>) -> PathBuf { pub fn new(account_config: &AccountConfig, folder: &str) -> Result<Self> {
let mut db_path = dir.as_ref().join(ID_MAPPER_DB_FILE_NAME); let digest = md5::compute(account_config.name.clone() + folder);
let mut db_parent_dir = dir.as_ref().parent(); let db_path = data_dir()
.ok_or(anyhow!("cannot get XDG data directory"))?
.join("himalaya")
.join(".id-mappers")
.join(format!("{digest:x}"));
while !db_path.is_file() { let conn = Config::new()
match db_parent_dir { .path(&db_path)
Some(dir) => { .idgen_persist_interval(1)
db_path = dir.join(ID_MAPPER_DB_FILE_NAME); .open()
db_parent_dir = dir.parent();
}
None => {
db_path = dir.as_ref().join(ID_MAPPER_DB_FILE_NAME);
break;
}
}
}
db_path
}
pub fn new(account_config: &AccountConfig, folder: &str, db_path: PathBuf) -> Result<Self> {
let folder = account_config.get_folder_alias(folder);
let digest = md5::compute(account_config.name.clone() + &folder);
let table = format!("id_mapper_{digest:x}");
debug!("creating id mapper table {table} at {db_path:?}…");
let db_path = Self::find_closest_db_path(db_path);
let conn = rusqlite::Connection::open(&db_path)
.with_context(|| format!("cannot open id mapper database at {db_path:?}"))?; .with_context(|| format!("cannot open id mapper database at {db_path:?}"))?;
let query = format!( Ok(Self::Mapper(conn))
"CREATE TABLE IF NOT EXISTS {table} (
id INTEGER PRIMARY KEY AUTOINCREMENT,
internal_id TEXT UNIQUE
)",
);
trace!("create table query: {query:#?}");
conn.execute(&query, [])
.context("cannot create id mapper table")?;
Ok(Self::Mapper(table, conn))
} }
pub fn create_alias<I>(&self, id: I) -> Result<String> pub fn create_alias<I>(&self, id: I) -> Result<String>
@ -66,18 +39,18 @@ impl IdMapper {
let id = id.as_ref(); let id = id.as_ref();
match self { match self {
Self::Dummy => Ok(id.to_owned()), Self::Dummy => Ok(id.to_owned()),
Self::Mapper(table, conn) => { Self::Mapper(conn) => {
debug!("creating alias for id {id}…"); debug!("creating alias for id {id}…");
let query = format!("INSERT OR IGNORE INTO {} (internal_id) VALUES (?)", table); let alias = conn
trace!("insert query: {query:#?}"); .generate_id()
.with_context(|| format!("cannot create alias for id {id}"))?
conn.execute(&query, [id]) .to_string();
.with_context(|| format!("cannot create id alias for id {id}"))?;
let alias = conn.last_insert_rowid().to_string();
debug!("created alias {alias} for id {id}"); debug!("created alias {alias} for id {id}");
conn.insert(&id, alias.as_bytes())
.with_context(|| format!("cannot insert alias {alias} for id {id}"))?;
Ok(alias) Ok(alias)
} }
} }
@ -90,22 +63,16 @@ impl IdMapper {
let id = id.as_ref(); let id = id.as_ref();
match self { match self {
Self::Dummy => Ok(id.to_owned()), Self::Dummy => Ok(id.to_owned()),
Self::Mapper(table, conn) => { Self::Mapper(conn) => {
debug!("getting alias for id {id}…"); debug!("getting alias for id {id}…");
let query = format!("SELECT id FROM {} WHERE internal_id = ?", table); let alias = conn
trace!("select query: {query:#?}"); .get(id)
.with_context(|| format!("cannot get alias for id {id}"))?;
let mut stmt = conn let alias = match alias {
.prepare(&query)
.with_context(|| format!("cannot get alias for id {id}"))?;
let aliases: Vec<i64> = stmt
.query_map([id], |row| row.get(0))
.with_context(|| format!("cannot get alias for id {id}"))?
.collect::<rusqlite::Result<_>>()
.with_context(|| format!("cannot get alias for id {id}"))?;
let alias = match aliases.first() {
Some(alias) => { Some(alias) => {
let alias = String::from_utf8_lossy(alias.as_ref());
debug!("found alias {alias} for id {id}"); debug!("found alias {alias} for id {id}");
alias.to_string() alias.to_string()
} }
@ -125,30 +92,24 @@ impl IdMapper {
A: ToString, A: ToString,
{ {
let alias = alias.to_string(); let alias = alias.to_string();
let alias = alias
.parse::<i64>()
.context(format!("cannot parse id mapper alias {alias}"))?;
match self { match self {
Self::Dummy => Ok(alias.to_string()), Self::Dummy => Ok(alias.to_string()),
Self::Mapper(table, conn) => { Self::Mapper(conn) => {
debug!("getting id from alias {alias}…"); debug!("getting id from alias {alias}…");
let query = format!("SELECT internal_id FROM {} WHERE id = ?", table); let id = conn
trace!("select query: {query:#?}"); .iter()
.flat_map(|entry| entry)
let mut stmt = conn .find_map(|(entry_id, entry_alias)| {
.prepare(&query) if entry_alias.as_ref() == alias.as_bytes() {
.with_context(|| format!("cannot get id from alias {alias}"))?; let entry_id = String::from_utf8_lossy(entry_id.as_ref());
let ids: Vec<String> = stmt Some(entry_id.to_string())
.query_map([alias], |row| row.get(0)) } else {
.with_context(|| format!("cannot get id from alias {alias}"))? None
.collect::<rusqlite::Result<_>>() }
.with_context(|| format!("cannot get id from alias {alias}"))?; })
let id = ids .ok_or_else(|| anyhow!("cannot get id from alias {alias}"))?;
.first()
.ok_or_else(|| anyhow!("cannot get id from alias {alias}"))?
.to_owned();
debug!("found id {id} from alias {alias}"); debug!("found id {id} from alias {alias}");
Ok(id) Ok(id)
@ -156,14 +117,29 @@ impl IdMapper {
} }
} }
pub fn get_ids<A, I>(&self, aliases: I) -> Result<Vec<String>> pub fn get_ids(&self, aliases: impl IntoIterator<Item = impl ToString>) -> Result<Vec<String>> {
where let aliases: Vec<String> = aliases.into_iter().map(|alias| alias.to_string()).collect();
A: ToString,
I: IntoIterator<Item = A>, match self {
{ Self::Dummy => Ok(aliases),
aliases Self::Mapper(conn) => {
.into_iter() let aliases: HashSet<&str> = aliases.iter().map(|alias| alias.as_str()).collect();
.map(|alias| self.get_id(alias)) let ids: Vec<String> = conn
.collect() .iter()
.flat_map(|entry| entry)
.filter_map(|(entry_id, entry_alias)| {
let alias = String::from_utf8_lossy(entry_alias.as_ref());
if aliases.contains(alias.as_ref()) {
let entry_id = String::from_utf8_lossy(entry_id.as_ref());
Some(entry_id.to_string())
} else {
None
}
})
.collect();
Ok(ids)
}
}
} }
} }