diff --git a/Cargo.lock b/Cargo.lock index a9a9b6685e..ddf8503abc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1595,6 +1595,18 @@ dependencies = [ "zune-inflate", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "faster-hex" version = "0.9.0" @@ -2129,6 +2141,7 @@ dependencies = [ "git2", "git2-hooks", "gitbutler-git", + "gitbutler-project-store", "gitbutler-testsupport", "gix", "glob", @@ -2198,6 +2211,15 @@ dependencies = [ "walkdir", ] +[[package]] +name = "gitbutler-project-store" +version = "0.0.0" +dependencies = [ + "anyhow", + "rusqlite", + "uuid", +] + [[package]] name = "gitbutler-tauri" version = "0.0.0" @@ -3151,6 +3173,15 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "hdrhistogram" version = "7.5.4" @@ -3859,6 +3890,17 @@ dependencies = [ "libc", ] +[[package]] +name = "libsqlite3-sys" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "libssh2-sys" version = "0.3.0" @@ -5577,6 +5619,21 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "rusqlite" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae" +dependencies = [ + "bitflags 2.6.0", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "uuid", +] + [[package]] name = "rust_decimal" version = "1.35.0" diff --git a/Cargo.toml b/Cargo.toml index 001ffb3022..f222ce23a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/gitbutler-watcher/vendor/debouncer", "crates/gitbutler-testsupport", "crates/gitbutler-cli", + "crates/gitbutler-project-store" ] resolver = "2" @@ -14,7 +15,7 @@ resolver = "2" # Add the `tracing` or `tracing-detail` features to see more of gitoxide in the logs. Useful to see which programs it invokes. gix = { git = "https://github.com/Byron/gitoxide", rev = "55cbc1b9d6f210298a86502a7f20f9736c7e963e", default-features = false, features = [] } git2 = { version = "0.18.3", features = ["vendored-openssl", "vendored-libgit2"] } -uuid = { version = "1.8.0", features = ["serde"] } +uuid = { version = "1.8.0", features = ["serde", "v4", "v7"] } serde = { version = "1.0", features = ["derive"] } thiserror = "1.0.61" tokio = { version = "1.38.0", default-features = false } @@ -25,6 +26,7 @@ gitbutler-core = { path = "crates/gitbutler-core" } gitbutler-watcher = { path = "crates/gitbutler-watcher" } gitbutler-testsupport = { path = "crates/gitbutler-testsupport" } gitbutler-cli ={ path = "crates/gitbutler-cli" } +gitbutler-project-store = { path = "crates/gitbutler-project-store" } [profile.release] codegen-units = 1 # Compile crates one after another so the compiler can optimize better diff --git a/crates/gitbutler-core/Cargo.toml b/crates/gitbutler-core/Cargo.toml index 8e2d255276..dad48da780 100644 --- a/crates/gitbutler-core/Cargo.toml +++ b/crates/gitbutler-core/Cargo.toml @@ -57,6 +57,7 @@ uuid.workspace = true walkdir = "2.5.0" zip = "0.6.5" gitbutler-git.workspace = true +gitbutler-project-store.workspace = true [features] # by default Tauri runs in production mode diff --git a/crates/gitbutler-project-store/Cargo.toml b/crates/gitbutler-project-store/Cargo.toml new file mode 100644 index 0000000000..9e55c0f238 --- /dev/null +++ b/crates/gitbutler-project-store/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "gitbutler-project-store" +version = "0.0.0" +edition = "2021" +publish = false + +[lib] +path = "src/lib.rs" +doctest = false +test = false + +[dev.dependencies] +gitbutler-testsupport.workspace = true + +[dependencies] +rusqlite = { version = "0.31.0", features = ["bundled", "uuid"] } +anyhow = "1.0.86" +uuid.workspace = true \ No newline at end of file diff --git a/crates/gitbutler-project-store/src/changes.rs b/crates/gitbutler-project-store/src/changes.rs new file mode 100644 index 0000000000..1121c57662 --- /dev/null +++ b/crates/gitbutler-project-store/src/changes.rs @@ -0,0 +1,9 @@ +use rusqlite::Connection; + +// TODO: rename to patches +/// The changes struct provides a +pub struct Changes<'l> { + connection: &'l mut Connection, +} + +impl<'l> Changes<'l> {} diff --git a/crates/gitbutler-project-store/src/lib.rs b/crates/gitbutler-project-store/src/lib.rs new file mode 100644 index 0000000000..05e75a82a8 --- /dev/null +++ b/crates/gitbutler-project-store/src/lib.rs @@ -0,0 +1,92 @@ +use anyhow::{anyhow, Context, Result}; +use migrations::{gitbutler_migrations::gitbutler_migrations, migrator::Migrator}; +use rusqlite::Connection; +use std::path::Path; + +mod changes; +mod migrations; + +const DATABASE_NAME: &str = "project.sqlite"; + +/// ProjectStore provides a light wrapper around a sqlite database +struct ProjectStore { + connection: Connection, +} + +impl ProjectStore {} + +/// Database setup +/// +/// Before touching any database related code, please read https://github.com/the-lean-crate/criner/discussions/5 first. +impl ProjectStore { + /// Creates an instance of ProjectStore and runs any pending sqlite migrations + /// gitbutler_project_directory should be the `.git/gitbutler` path of a given + /// repository + pub fn initialize(gitbutler_project_directory: &Path) -> Result { + let database_path = gitbutler_project_directory.join(DATABASE_NAME); + let database_path = database_path.to_str().ok_or(anyhow!( + "Failed to get database {}", + gitbutler_project_directory.display() + ))?; + + let connection = Connection::open(database_path)?; + + ProjectStore::configure_connection(&connection)?; + + let mut project_store = ProjectStore { connection }; + + project_store.run_migrations()?; + + Ok(project_store) + } + + /// Configures a sqlite connection to behave sensibly in a concurrent environemnt. + /// + /// Busy handler and pargma's have been taken from https://github.com/the-lean-crate/criner/discussions/5 + /// and will help with concurrent reading and writing. + /// + /// This should be run before a project store is created and any other SQL is run. + fn configure_connection(connection: &Connection) -> Result<()> { + connection + .busy_handler(Some(sleeper)) + .context("Failed to set connection's busy handler")?; + + connection.execute_batch(" + PRAGMA journal_mode = WAL; -- better write-concurrency + PRAGMA synchronous = NORMAL; -- fsync only in critical moments + PRAGMA wal_autocheckpoint = 1000; -- write WAL changes back every 1000 pages, for an in average 1MB WAL file. May affect readers if number is increased + PRAGMA wal_checkpoint(TRUNCATE); -- free some space by truncating possibly massive WAL files from the last run. + ").context("Failed to set PRAGMA's for connection")?; + + Ok(()) + } + + /// Calls the migrator with appropriate migrations + fn run_migrations(&mut self) -> Result<()> { + let mut migrator = Migrator::new(&mut self.connection); + migrator.migrate(gitbutler_migrations())?; + Ok(()) + } +} + +fn sleeper(attempts: i32) -> bool { + println!("SQLITE_BUSY, retrying after 50ms (attempt {})", attempts); + std::thread::sleep(std::time::Duration::from_millis(50)); + true +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn run_migrations_should_succeed() { + let connection = Connection::open_in_memory().unwrap(); + + ProjectStore::configure_connection(&connection).unwrap(); + + let mut project_store = ProjectStore { connection }; + + project_store.run_migrations().unwrap() + } +} diff --git a/crates/gitbutler-project-store/src/migrations/gitbutler_migrations.rs b/crates/gitbutler-project-store/src/migrations/gitbutler_migrations.rs new file mode 100644 index 0000000000..2964ebf358 --- /dev/null +++ b/crates/gitbutler-project-store/src/migrations/gitbutler_migrations.rs @@ -0,0 +1,29 @@ +use super::migration::Migration; + +const CREATE_BASE_SCHEMA: &str = " +CREATE TABLE changes ( + id BLOB PRIMARY KEY NOT NULL, -- A UUID v4 + is_unapplied_wip INTEGER NOT NULL, + unapplied_vbranch_name TEXT, + created_at INTEGER -- A unix timestamp in seconds when the record was created +); +CREATE TABLE commits ( + sha TEXT PRIMARY KEY NOT NULL, -- A commit SHA as a base16 string + created_at INTEGER, -- A unix timestamp in seconds when the record was created + change_id BLOB NOT NULL, + FOREIGN KEY(change_id) REFERENCES changes(change_id) +); +"; + +pub(crate) fn gitbutler_migrations() -> Vec { + let base_migration = Migration { + name: "base".to_string(), + up: |connection| { + connection.execute_batch(CREATE_BASE_SCHEMA)?; + + Ok(()) + }, + }; + + vec![base_migration] +} diff --git a/crates/gitbutler-project-store/src/migrations/migration.rs b/crates/gitbutler-project-store/src/migrations/migration.rs new file mode 100644 index 0000000000..9a883e639d --- /dev/null +++ b/crates/gitbutler-project-store/src/migrations/migration.rs @@ -0,0 +1,10 @@ +use anyhow::Result; +use rusqlite::Connection; + +pub(crate) struct Migration { + /// A unique identifier for the migration + pub name: String, + /// A function which performs the migration. The up function gets run inside + /// of a transaction. + pub up: fn(&Connection) -> Result<()>, +} diff --git a/crates/gitbutler-project-store/src/migrations/migrator.rs b/crates/gitbutler-project-store/src/migrations/migrator.rs new file mode 100644 index 0000000000..4bb852141e --- /dev/null +++ b/crates/gitbutler-project-store/src/migrations/migrator.rs @@ -0,0 +1,204 @@ +use anyhow::{Context, Ok, Result}; +use rusqlite::{Connection, TransactionBehavior}; +use uuid::Uuid; + +use super::migration::Migration; + +/// SQL required to create the migrations table. +/// +/// This should only ever be changed with great caution +const CREATE_MIGRATIONS_TABLE: &str = " +CREATE TABLE IF NOT EXISTS migrations ( + id BLOB PRIMARY KEY NOT NULL, -- A V4 UUID + name TEXT NOT NULL +) +"; + +/// Migrator is used to perform migrations on a particular database. +pub(crate) struct Migrator<'l> { + connection: &'l mut Connection, +} + +impl<'l> Migrator<'l> { + pub(crate) fn new(connection: &'l mut Connection) -> Migrator<'l> { + Migrator { connection } + } + + /// Iterates over the list of provided migrations starting at index 0. + /// If the migration has already been run, it will be skipped. + /// Run migrations get recorded in the `migrations` table. + /// The `migrations` table will be created if it doesn't exist. + pub(crate) fn migrate(&mut self, migrations: Vec) -> Result<()> { + let applied_migrations = self.find_applied_migrations()?; + + for migration in migrations { + // Don't try to reapply existing migrations + if applied_migrations.contains(&migration.name) { + continue; + } + + // Scope for transaction + { + // Using an Immediate transactions as both reads and writes + // may be performed in a migration. + let transaction = self + .connection + .transaction_with_behavior(TransactionBehavior::Immediate)?; + (migration.up)(&transaction) + .context(format!("Failed to run migration {}", migration.name))?; + + transaction + .execute( + "INSERT INTO migrations (id, name) VALUES (?1, ?2)", + (Uuid::new_v4(), migration.name), + ) + .context("Failed to insert migration completion marker")?; + + transaction.commit()?; + } + } + + Ok(()) + } + + /// Queries all the applied migrations. + /// If the `migrations` table doesn't exist, it will be created here + fn find_applied_migrations(&self) -> Result> { + self.connection + .execute(CREATE_MIGRATIONS_TABLE, []) + .context("Failed to create migrations table")?; + + let mut statement = self + .connection + .prepare("SELECT name FROM migrations") + .context("Failed to fetch migrations")?; + + let mapped_rows = statement + .query_map([], |row| row.get(0))? + .collect::, _>>()?; + + Ok(mapped_rows) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn first_migration() -> Migration { + Migration { + name: "first".to_string(), + up: |connection| { + connection.execute( + " + CREATE TABLE testaroni ( + id BLOB PRIMARY KEY, -- A V4 UUID + name TEXT NOT NULL + ) + ", + [], + )?; + + Ok(()) + }, + } + } + + /// Depends on `first_migration` + fn second_migration() -> Migration { + Migration { + name: "second".to_string(), + up: |connection| { + connection.execute("ALTER TABLE testaroni ADD potatoes TEXT", [])?; + + Ok(()) + }, + } + } + + #[test] + /// Testing the `find_applied_migrations` function. + /// Ensuring that it creates the `migrations` table if it doesn't exist + fn find_applied_migrations_creates_migrations_table_if_not_exists() { + let mut connection = Connection::open_in_memory().unwrap(); + + let migrator = Migrator::new(&mut connection); + + // Try to find unapplied migrations + migrator.find_applied_migrations().unwrap(); + + let statement = connection + .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='migrations'") + .unwrap(); + assert_eq!(statement.column_count(), 1); + } + + #[test] + /// Testing the `find_applied_migrations` function. + /// Assuming the `migrations` table exists (which we create by calling + /// `find_applied_migrations), we expect that any values inserted into + /// the table will get returned when we call it again. + fn find_applied_migrations_lists_all_entries_in_table() { + let mut connection = Connection::open_in_memory().unwrap(); + + // Call find_applied_migrations in order to create the migrations table + { + let migrator = Migrator::new(&mut connection); + migrator.find_applied_migrations().unwrap(); + } + + // Insert some entries into the migrations table + connection + .execute( + "INSERT INTO migrations (id, name) VALUES (?1, ?2)", + (Uuid::new_v4(), "base".to_string()), + ) + .unwrap(); + + connection + .execute( + "INSERT INTO migrations (id, name) VALUES (?1, ?2)", + (Uuid::new_v4(), "other".to_string()), + ) + .unwrap(); + + { + let migrator = Migrator::new(&mut connection); + let results = migrator.find_applied_migrations().unwrap(); + assert_eq!(results.len(), 2); + assert!(results.contains(&"base".to_string())); + assert!(results.contains(&"other".to_string())); + } + } + + #[test] + /// Testing the `migrate` function. + /// When given a list of migrations to run, it will perform the migrations in order + fn migrate_applies_migrations_in_order() { + let mut connection = Connection::open_in_memory().unwrap(); + + let mut migrator = Migrator::new(&mut connection); + // Runs two migrations, one which creates a table and the other which laters it + migrator + .migrate(vec![first_migration(), second_migration()]) + .unwrap(); + } + + #[test] + /// Testing the `migrate` function. + /// When calling multiple times on the same database, it will only preform + /// migrations that haven't been run before. + /// + /// Given the provided migrations, we would expect the second call to + /// `migrate` to return Err if it was run twice. + fn migrate_only_applies_migrations_once() { + let mut connection = Connection::open_in_memory().unwrap(); + + let mut migrator = Migrator::new(&mut connection); + migrator.migrate(vec![first_migration()]).unwrap(); + + migrator + .migrate(vec![first_migration(), second_migration()]) + .unwrap(); + } +} diff --git a/crates/gitbutler-project-store/src/migrations/mod.rs b/crates/gitbutler-project-store/src/migrations/mod.rs new file mode 100644 index 0000000000..38532df94e --- /dev/null +++ b/crates/gitbutler-project-store/src/migrations/mod.rs @@ -0,0 +1,3 @@ +pub mod gitbutler_migrations; +pub mod migration; +pub mod migrator;