diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index dbf6dd7f..0a18d382 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -1,8 +1,6 @@ use std::{fmt::Display, rc::Rc}; use limbo_core::{Connection, Result, StepResult}; -use rand::SeedableRng; -use rand_chacha::ChaCha8Rng; use crate::{ model::{ @@ -19,20 +17,56 @@ use super::{pick, pick_index}; pub(crate) type ResultSet = Result>>; pub(crate) struct InteractionPlan { - pub(crate) plan: Vec, + pub(crate) plan: Vec, pub(crate) stack: Vec, pub(crate) interaction_pointer: usize, + pub(crate) secondary_pointer: usize, +} + +pub(crate) struct Property { + pub(crate) name: Option, + pub(crate) interactions: Vec, +} + +impl Property { + pub(crate) fn new(name: Option, interactions: Vec) -> Self { + Self { name, interactions } + } + + pub(crate) fn anonymous(interactions: Vec) -> Self { + Self { + name: None, + interactions, + } + } } impl Display for InteractionPlan { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - for interaction in &self.plan { - match interaction { - Interaction::Query(query) => writeln!(f, "{};", query)?, - Interaction::Assertion(assertion) => { - writeln!(f, "-- ASSERT: {};", assertion.message)? + for property in &self.plan { + if let Some(name) = &property.name { + writeln!(f, "-- begin testing '{}'", name)?; + } + + for interaction in &property.interactions { + if property.name.is_some() { + write!(f, "\t")?; } - Interaction::Fault(fault) => writeln!(f, "-- FAULT: {};", fault)?, + + match interaction { + Interaction::Query(query) => writeln!(f, "{};", query)?, + Interaction::Assumption(assumption) => { + writeln!(f, "-- ASSUME: {};", assumption.message)? + } + Interaction::Assertion(assertion) => { + writeln!(f, "-- ASSERT: {};", assertion.message)? + } + Interaction::Fault(fault) => writeln!(f, "-- FAULT: {};", fault)?, + } + } + + if let Some(name) = &property.name { + writeln!(f, "-- end testing '{}'", name)?; } } @@ -60,6 +94,7 @@ impl Display for InteractionStats { pub(crate) enum Interaction { Query(Query), + Assumption(Assertion), Assertion(Assertion), Fault(Fault), } @@ -68,13 +103,14 @@ impl Display for Interaction { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::Query(query) => write!(f, "{}", query), + Self::Assumption(assumption) => write!(f, "ASSUME: {}", assumption.message), Self::Assertion(assertion) => write!(f, "ASSERT: {}", assertion.message), Self::Fault(fault) => write!(f, "FAULT: {}", fault), } } } -type AssertionFunc = dyn Fn(&Vec) -> bool; +type AssertionFunc = dyn Fn(&Vec, &SimulatorEnv) -> bool; pub(crate) struct Assertion { pub(crate) func: Box, @@ -93,11 +129,9 @@ impl Display for Fault { } } -pub(crate) struct Interactions(Vec); - -impl Interactions { +impl Property { pub(crate) fn shadow(&self, env: &mut SimulatorEnv) { - for interaction in &self.0 { + for interaction in &self.interactions { match interaction { Interaction::Query(query) => match query { Query::Create(create) => { @@ -117,6 +151,7 @@ impl Interactions { Query::Select(_) => {} }, Interaction::Assertion(_) => {} + Interaction::Assumption(_) => {} Interaction::Fault(_) => {} } } @@ -129,29 +164,29 @@ impl InteractionPlan { plan: Vec::new(), stack: Vec::new(), interaction_pointer: 0, + secondary_pointer: 0, } } - pub(crate) fn push(&mut self, interaction: Interaction) { - self.plan.push(interaction); - } - pub(crate) fn stats(&self) -> InteractionStats { let mut read = 0; let mut write = 0; let mut delete = 0; let mut create = 0; - for interaction in &self.plan { - match interaction { - Interaction::Query(query) => match query { - Query::Select(_) => read += 1, - Query::Insert(_) => write += 1, - Query::Delete(_) => delete += 1, - Query::Create(_) => create += 1, - }, - Interaction::Assertion(_) => {} - Interaction::Fault(_) => {} + for property in &self.plan { + for interaction in &property.interactions { + match interaction { + Interaction::Query(query) => match query { + Query::Select(_) => read += 1, + Query::Insert(_) => write += 1, + Query::Delete(_) => delete += 1, + Query::Create(_) => create += 1, + }, + Interaction::Assertion(_) => {} + Interaction::Assumption(_) => {} + Interaction::Fault(_) => {} + } } } @@ -164,25 +199,22 @@ impl InteractionPlan { } } -impl ArbitraryFrom for InteractionPlan { - fn arbitrary_from(rng: &mut R, env: &SimulatorEnv) -> Self { +impl InteractionPlan { + // todo: This is a hack to get around the fact that `ArbitraryFrom` can't take a mutable + // reference of T, so instead write a bespoke function without using the trait system. + pub(crate) fn arbitrary_from(rng: &mut R, env: &mut SimulatorEnv) -> Self { let mut plan = InteractionPlan::new(); - let mut env = SimulatorEnv { - opts: env.opts.clone(), - tables: vec![], - connections: vec![], - io: env.io.clone(), - db: env.db.clone(), - rng: ChaCha8Rng::seed_from_u64(rng.next_u64()), - }; - let num_interactions = env.opts.max_interactions; // First create at least one table let create_query = Create::arbitrary(rng); env.tables.push(create_query.table.clone()); - plan.push(Interaction::Query(Query::Create(create_query))); + + plan.plan.push(Property { + name: Some("initial table creation".to_string()), + interactions: vec![Interaction::Query(Query::Create(create_query))], + }); while plan.plan.len() < num_interactions { log::debug!( @@ -190,10 +222,10 @@ impl ArbitraryFrom for InteractionPlan { plan.plan.len(), num_interactions ); - let interactions = Interactions::arbitrary_from(rng, &(&env, plan.stats())); - interactions.shadow(&mut env); + let property = Property::arbitrary_from(rng, &(env, plan.stats())); + property.shadow(env); - plan.plan.extend(interactions.0.into_iter()); + plan.plan.push(property); } log::info!("Generated plan with {} interactions", plan.plan.len()); @@ -251,31 +283,67 @@ impl Interaction { Self::Assertion(_) => { unreachable!("unexpected: this function should only be called on queries") } - Interaction::Fault(_) => { + Self::Assumption(_) => { + unreachable!("unexpected: this function should only be called on queries") + } + Self::Fault(_) => { unreachable!("unexpected: this function should only be called on queries") } } } - pub(crate) fn execute_assertion(&self, stack: &Vec) -> Result<()> { + pub(crate) fn execute_assertion( + &self, + stack: &Vec, + env: &SimulatorEnv, + ) -> Result<()> { match self { Self::Query(_) => { unreachable!("unexpected: this function should only be called on assertions") } Self::Assertion(assertion) => { - if !assertion.func.as_ref()(stack) { + if !assertion.func.as_ref()(stack, env) { return Err(limbo_core::LimboError::InternalError( assertion.message.clone(), )); } Ok(()) } + Self::Assumption(_) => { + unreachable!("unexpected: this function should only be called on assertions") + } Self::Fault(_) => { unreachable!("unexpected: this function should only be called on assertions") } } } + pub(crate) fn execute_assumption( + &self, + stack: &Vec, + env: &SimulatorEnv, + ) -> Result<()> { + match self { + Self::Query(_) => { + unreachable!("unexpected: this function should only be called on assumptions") + } + Self::Assertion(_) => { + unreachable!("unexpected: this function should only be called on assumptions") + } + Self::Assumption(assumption) => { + if !assumption.func.as_ref()(stack, env) { + return Err(limbo_core::LimboError::InternalError( + assumption.message.clone(), + )); + } + Ok(()) + } + Self::Fault(_) => { + unreachable!("unexpected: this function should only be called on assumptions") + } + } + } + pub(crate) fn execute_fault(&self, env: &mut SimulatorEnv, conn_index: usize) -> Result<()> { match self { Self::Query(_) => { @@ -284,6 +352,9 @@ impl Interaction { Self::Assertion(_) => { unreachable!("unexpected: this function should only be called on faults") } + Self::Assumption(_) => { + unreachable!("unexpected: this function should only be called on faults") + } Self::Fault(fault) => { match fault { Fault::Disconnect => { @@ -306,7 +377,7 @@ impl Interaction { } } -fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Interactions { +fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Property { // Get a random table let table = pick(&env.tables, rng); // Pick a random column @@ -324,6 +395,18 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Inte row.push(value); } } + + // Check that the table exists + let assumption = Interaction::Assumption(Assertion { + message: format!("table {} exists", table.name), + func: Box::new({ + let table_name = table.name.clone(); + move |_: &Vec, env: &SimulatorEnv| { + env.tables.iter().any(|t| t.name == table_name) + } + }), + }); + // Insert the row let insert_query = Interaction::Query(Query::Insert(Insert { table: table.name.clone(), @@ -345,7 +428,7 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Inte column.name, value, ), - func: Box::new(move |stack: &Vec| { + func: Box::new(move |stack: &Vec, _: &SimulatorEnv| { let rows = stack.last().unwrap(); match rows { Ok(rows) => rows.iter().any(|r| r == &row), @@ -354,10 +437,13 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Inte }), }); - Interactions(vec![insert_query, select_query, assertion]) + Property::new( + Some("select contains inserted value".to_string()), + vec![assumption, insert_query, select_query, assertion], + ) } -fn property_double_create_failure(rng: &mut R, _env: &SimulatorEnv) -> Interactions { +fn property_double_create_failure(rng: &mut R, _env: &SimulatorEnv) -> Property { let create_query = Create::arbitrary(rng); let table_name = create_query.table.name.clone(); let cq1 = Interaction::Query(Query::Create(create_query.clone())); @@ -367,7 +453,7 @@ fn property_double_create_failure(rng: &mut R, _env: &SimulatorEnv message: "creating two tables with the name should result in a failure for the second query" .to_string(), - func: Box::new(move |stack: &Vec| { + func: Box::new(move |stack: &Vec, _: &SimulatorEnv| { let last = stack.last().unwrap(); match last { Ok(_) => false, @@ -378,31 +464,34 @@ fn property_double_create_failure(rng: &mut R, _env: &SimulatorEnv }), }); - Interactions(vec![cq1, cq2, assertion]) + Property::new( + Some("creating the same table twice fails".to_string()), + vec![cq1, cq2, assertion], + ) } -fn create_table(rng: &mut R, _env: &SimulatorEnv) -> Interactions { +fn create_table(rng: &mut R, _env: &SimulatorEnv) -> Property { let create_query = Interaction::Query(Query::Create(Create::arbitrary(rng))); - Interactions(vec![create_query]) + Property::anonymous(vec![create_query]) } -fn random_read(rng: &mut R, env: &SimulatorEnv) -> Interactions { +fn random_read(rng: &mut R, env: &SimulatorEnv) -> Property { let select_query = Interaction::Query(Query::Select(Select::arbitrary_from(rng, &env.tables))); - Interactions(vec![select_query]) + Property::anonymous(vec![select_query]) } -fn random_write(rng: &mut R, env: &SimulatorEnv) -> Interactions { +fn random_write(rng: &mut R, env: &SimulatorEnv) -> Property { let table = pick(&env.tables, rng); let insert_query = Interaction::Query(Query::Insert(Insert::arbitrary_from(rng, table))); - Interactions(vec![insert_query]) + Property::anonymous(vec![insert_query]) } -fn random_fault(_rng: &mut R, _env: &SimulatorEnv) -> Interactions { +fn random_fault(_rng: &mut R, _env: &SimulatorEnv) -> Property { let fault = Interaction::Fault(Fault::Disconnect); - Interactions(vec![fault]) + Property::anonymous(vec![fault]) } -impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions { +impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Property { fn arbitrary_from( rng: &mut R, (env, stats): &(&SimulatorEnv, InteractionStats), diff --git a/simulator/main.rs b/simulator/main.rs index 52c33d5e..5491c578 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -1,8 +1,9 @@ #![allow(clippy::arc_with_non_send_sync, dead_code)] use clap::Parser; +use core::panic; +use generation::pick_index; use generation::plan::{Interaction, InteractionPlan, ResultSet}; -use generation::{pick_index, ArbitraryFrom}; -use limbo_core::{Database, Result}; +use limbo_core::{Database, LimboError, Result}; use model::table::Value; use rand::prelude::*; use rand_chacha::ChaCha8Rng; @@ -36,10 +37,12 @@ fn main() { let db_path = output_dir.join("simulator.db"); let plan_path = output_dir.join("simulator.plan"); + let history_path = output_dir.join("simulator.history"); // Print the seed, the locations of the database and the plan file log::info!("database path: {:?}", db_path); log::info!("simulator plan path: {:?}", plan_path); + log::info!("simulator history path: {:?}", history_path); log::info!("seed: {}", seed); std::panic::set_hook(Box::new(move |info| { @@ -73,28 +76,34 @@ fn main() { std::panic::catch_unwind(|| run_simulation(seed, &cli_opts, &db_path, &plan_path)); match (result, result2) { - (Ok(Ok(_)), Err(_)) => { + (Ok(ExecutionResult { error: None, .. }), Err(_)) => { log::error!("doublecheck failed! first run succeeded, but second run panicked."); } - (Ok(Err(_)), Err(_)) => { + (Ok(ExecutionResult { error: Some(_), .. }), Err(_)) => { log::error!( "doublecheck failed! first run failed assertion, but second run panicked." ); } - (Err(_), Ok(Ok(_))) => { + (Err(_), Ok(ExecutionResult { error: None, .. })) => { log::error!("doublecheck failed! first run panicked, but second run succeeded."); } - (Err(_), Ok(Err(_))) => { + (Err(_), Ok(ExecutionResult { error: Some(_), .. })) => { log::error!( "doublecheck failed! first run panicked, but second run failed assertion." ); } - (Ok(Ok(_)), Ok(Err(_))) => { + ( + Ok(ExecutionResult { error: None, .. }), + Ok(ExecutionResult { error: Some(_), .. }), + ) => { log::error!( "doublecheck failed! first run succeeded, but second run failed assertion." ); } - (Ok(Err(_)), Ok(Ok(_))) => { + ( + Ok(ExecutionResult { error: Some(_), .. }), + Ok(ExecutionResult { error: None, .. }), + ) => { log::error!( "doublecheck failed! first run failed assertion, but second run succeeded." ); @@ -122,18 +131,32 @@ fn main() { std::fs::rename(&old_db_path, &db_path).unwrap(); std::fs::rename(&old_plan_path, &plan_path).unwrap(); } else if let Ok(result) = result { - match result { - Ok(_) => { + // No panic occurred, so write the history to a file + let f = std::fs::File::create(&history_path).unwrap(); + let mut f = std::io::BufWriter::new(f); + for execution in result.history.history.iter() { + writeln!( + f, + "{} {} {}", + execution.connection_index, execution.interaction_index, execution.secondary_index + ) + .unwrap(); + } + + match result.error { + None => { log::info!("simulation completed successfully"); } - Err(e) => { + Some(e) => { log::error!("simulation failed: {:?}", e); } } } + // Print the seed, the locations of the database and the plan file at the end again for easily accessing them. println!("database path: {:?}", db_path); println!("simulator plan path: {:?}", plan_path); + println!("simulator history path: {:?}", history_path); println!("seed: {}", seed); } @@ -142,7 +165,7 @@ fn run_simulation( cli_opts: &SimulatorCLI, db_path: &Path, plan_path: &Path, -) -> Result<()> { +) -> ExecutionResult { let mut rng = ChaCha8Rng::seed_from_u64(seed); let (create_percent, read_percent, write_percent, delete_percent) = { @@ -160,21 +183,15 @@ fn run_simulation( }; if cli_opts.minimum_size < 1 { - return Err(limbo_core::LimboError::InternalError( - "minimum size must be at least 1".to_string(), - )); + panic!("minimum size must be at least 1"); } if cli_opts.maximum_size < 1 { - return Err(limbo_core::LimboError::InternalError( - "maximum size must be at least 1".to_string(), - )); + panic!("maximum size must be at least 1"); } if cli_opts.maximum_size < cli_opts.minimum_size { - return Err(limbo_core::LimboError::InternalError( - "maximum size must be greater than or equal to minimum size".to_string(), - )); + panic!("maximum size must be greater than or equal to minimum size"); } let opts = SimulatorOpts { @@ -212,7 +229,7 @@ fn run_simulation( log::info!("Generating database interaction plan..."); let mut plans = (1..=env.opts.max_connections) - .map(|_| InteractionPlan::arbitrary_from(&mut env.rng.clone(), &env)) + .map(|_| InteractionPlan::arbitrary_from(&mut env.rng.clone(), &mut env)) .collect::>(); let mut f = std::fs::File::create(plan_path).unwrap(); @@ -224,9 +241,6 @@ fn run_simulation( log::info!("Executing database interaction plan..."); let result = execute_plans(&mut env, &mut plans); - if result.is_err() { - log::error!("error executing plans: {:?}", result.as_ref().err()); - } env.io.print_stats(); @@ -235,23 +249,76 @@ fn run_simulation( result } -fn execute_plans(env: &mut SimulatorEnv, plans: &mut [InteractionPlan]) -> Result<()> { +struct Execution { + connection_index: usize, + interaction_index: usize, + secondary_index: usize, +} + +impl Execution { + fn new(connection_index: usize, interaction_index: usize, secondary_index: usize) -> Self { + Self { + connection_index, + interaction_index, + secondary_index, + } + } +} + +struct ExecutionHistory { + history: Vec, +} + +impl ExecutionHistory { + fn new() -> Self { + Self { + history: Vec::new(), + } + } +} + +struct ExecutionResult { + history: ExecutionHistory, + error: Option, +} + +impl ExecutionResult { + fn new(history: ExecutionHistory, error: Option) -> Self { + Self { history, error } + } +} + +fn execute_plans(env: &mut SimulatorEnv, plans: &mut [InteractionPlan]) -> ExecutionResult { + let mut history = ExecutionHistory::new(); let now = std::time::Instant::now(); // todo: add history here by recording which interaction was executed at which tick for _tick in 0..env.opts.ticks { // Pick the connection to interact with let connection_index = pick_index(env.connections.len(), &mut env.rng); + history.history.push(Execution::new( + connection_index, + plans[connection_index].interaction_pointer, + plans[connection_index].secondary_pointer, + )); // Execute the interaction for the selected connection - execute_plan(env, connection_index, plans)?; + match execute_plan(env, connection_index, plans) { + Ok(_) => {} + Err(err) => { + return ExecutionResult::new(history, Some(err)); + } + } // Check if the maximum time for the simulation has been reached if now.elapsed().as_secs() >= env.opts.max_time_simulation as u64 { - return Err(limbo_core::LimboError::InternalError( - "maximum time for simulation reached".into(), - )); + return ExecutionResult::new( + history, + Some(limbo_core::LimboError::InternalError( + "maximum time for simulation reached".into(), + )), + ); } } - Ok(()) + ExecutionResult::new(history, None) } fn execute_plan( @@ -266,16 +333,35 @@ fn execute_plan( return Ok(()); } - let interaction = &plan.plan[plan.interaction_pointer]; + let interaction = &plan.plan[plan.interaction_pointer].interactions[plan.secondary_pointer]; if let SimConnection::Disconnected = connection { log::info!("connecting {}", connection_index); env.connections[connection_index] = SimConnection::Connected(env.db.connect()); } else { match execute_interaction(env, connection_index, interaction, &mut plan.stack) { - Ok(_) => { + Ok(next_execution) => { log::debug!("connection {} processed", connection_index); - plan.interaction_pointer += 1; + // Move to the next interaction or property + match next_execution { + ExecutionContinuation::NextInteraction => { + if plan.secondary_pointer + 1 + >= plan.plan[plan.interaction_pointer].interactions.len() + { + // If we have reached the end of the interactions for this property, move to the next property + plan.interaction_pointer += 1; + plan.secondary_pointer = 0; + } else { + // Otherwise, move to the next interaction + plan.secondary_pointer += 1; + } + } + ExecutionContinuation::NextProperty => { + // Skip to the next property + plan.interaction_pointer += 1; + plan.secondary_pointer = 0; + } + } } Err(err) => { log::error!("error {}", err); @@ -287,12 +373,23 @@ fn execute_plan( Ok(()) } +/// The next point of control flow after executing an interaction. +/// `execute_interaction` uses this type in conjunction with a result, where +/// the `Err` case indicates a full-stop due to a bug, and the `Ok` case +/// indicates the next step in the plan. +enum ExecutionContinuation { + /// Default continuation, execute the next interaction. + NextInteraction, + /// Typically used in the case of preconditions failures, skip to the next property. + NextProperty, +} + fn execute_interaction( env: &mut SimulatorEnv, connection_index: usize, interaction: &Interaction, stack: &mut Vec, -) -> Result<()> { +) -> Result { log::info!("executing: {}", interaction); match interaction { generation::plan::Interaction::Query(_) => { @@ -307,15 +404,24 @@ fn execute_interaction( stack.push(results); } generation::plan::Interaction::Assertion(_) => { - interaction.execute_assertion(stack)?; + interaction.execute_assertion(stack, env)?; stack.clear(); } + generation::plan::Interaction::Assumption(_) => { + let assumption_result = interaction.execute_assumption(stack, env); + stack.clear(); + + if assumption_result.is_err() { + log::warn!("assumption failed: {:?}", assumption_result); + return Ok(ExecutionContinuation::NextProperty); + } + } Interaction::Fault(_) => { interaction.execute_fault(env, connection_index)?; } } - Ok(()) + Ok(ExecutionContinuation::NextInteraction) } fn compare_equal_rows(a: &[Vec], b: &[Vec]) {