From 5378651df6417c4529c2e50e94114c6f1d6e4d5d Mon Sep 17 00:00:00 2001 From: Julian Gonzalez Calderon Date: Mon, 9 Dec 2024 16:35:32 -0300 Subject: [PATCH] Add benchmarks comparing to Cairo VM (#692) * Add benches * Fix clippy * Add more cases * Use fibonacci instead of fib * Differenciate expect messages * Separate into groups per program * Add result checks * Fix clippy * Add compile bench * Add linear search * Add linear search bench * Lower time * Fix program * Add linear_search.c * Add builtin * Fix * Fix symbol number * Update benches * Fix bench * Remove comp benchmark * Remove extra dependency --------- Co-authored-by: Edgar Co-authored-by: Pedro Fontana --- benches/benches.rs | 241 ++++++++++++++++++++++++--- programs/benches/linear_search.c | 37 ++++ programs/benches/linear_search.cairo | 37 ++++ 3 files changed, 295 insertions(+), 20 deletions(-) create mode 100644 programs/benches/linear_search.c create mode 100644 programs/benches/linear_search.cairo diff --git a/benches/benches.rs b/benches/benches.rs index b88a6f315..92bd300ab 100644 --- a/benches/benches.rs +++ b/benches/benches.rs @@ -1,12 +1,16 @@ use cairo_lang_compiler::{ compile_prepared_db, db::RootDatabase, project::setup_project, CompilerConfig, }; +use cairo_lang_runner::{RunResultValue, SierraCasmRunner, StarknetState}; use cairo_lang_sierra::program::Program; +use cairo_lang_sierra_generator::replace_ids::DebugReplacer; +use cairo_lang_starknet::contract::{find_contracts, get_contracts_info}; +use cairo_lang_utils::Upcast; use cairo_native::{ cache::{AotProgramCache, JitProgramCache}, context::NativeContext, utils::find_function_id, - OptLevel, + OptLevel, Value, }; use criterion::{criterion_group, criterion_main, Criterion}; use starknet_types_core::felt::Felt; @@ -20,6 +24,7 @@ fn criterion_benchmark(c: &mut Criterion) { let factorial = load_contract("programs/benches/factorial_2M.cairo"); let fibonacci = load_contract("programs/benches/fib_2M.cairo"); let logistic_map = load_contract("programs/benches/logistic_map.cairo"); + let linear_search = load_contract("programs/benches/linear_search.cairo"); let aot_factorial = aot_cache .compile_and_insert(Felt::ZERO, &factorial, OptLevel::Aggressive) @@ -30,6 +35,9 @@ fn criterion_benchmark(c: &mut Criterion) { let aot_logistic_map = aot_cache .compile_and_insert(Felt::from(2), &logistic_map, OptLevel::Aggressive) .unwrap(); + let aot_linear_search = aot_cache + .compile_and_insert(Felt::from(2), &linear_search, OptLevel::Aggressive) + .unwrap(); let jit_factorial = jit_cache .compile_and_insert(Felt::ZERO, &factorial, OptLevel::Aggressive) @@ -40,32 +48,195 @@ fn criterion_benchmark(c: &mut Criterion) { let jit_logistic_map = jit_cache .compile_and_insert(Felt::from(2), &logistic_map, OptLevel::Aggressive) .unwrap(); + let jit_linear_search = jit_cache + .compile_and_insert(Felt::from(2), &linear_search, OptLevel::Aggressive) + .unwrap(); let factorial_function_id = find_function_id(&factorial, "factorial_2M::factorial_2M::main").unwrap(); let fibonacci_function_id = find_function_id(&fibonacci, "fib_2M::fib_2M::main").unwrap(); let logistic_map_function_id = find_function_id(&logistic_map, "logistic_map::logistic_map::main").unwrap(); + let linear_search_function_id = + find_function_id(&linear_search, "linear_search::linear_search::main").unwrap(); + + let factorial_runner = load_contract_for_vm("programs/benches/factorial_2M.cairo"); + let fibonacci_runner = load_contract_for_vm("programs/benches/fib_2M.cairo"); + let logistic_map_runner = load_contract_for_vm("programs/benches/logistic_map.cairo"); + let linear_search_runner = load_contract_for_vm("programs/benches/linear_search.cairo"); + + let factorial_function = factorial_runner + .find_function("main") + .expect("failed to find main factorial function"); + let fibonacci_function = fibonacci_runner + .find_function("main") + .expect("failed to find main fibonacci function"); + let logistic_map_function = logistic_map_runner + .find_function("main") + .expect("failed to find main logistic map function"); + let linear_search_function = linear_search_runner + .find_function("main") + .expect("failed to find main logistic map function"); + + { + let mut linear_search_group = c.benchmark_group("linear_search"); + + linear_search_group.bench_function("Cached JIT", |b| { + b.iter(|| { + let result = jit_linear_search + .invoke_dynamic(linear_search_function_id, &[], Some(u64::MAX)) + .unwrap(); + let value = result.return_value; + assert!(matches!(value, Value::Enum { tag: 0, .. })) + }); + }); + linear_search_group.bench_function("Cached AOT", |b| { + b.iter(|| { + let result = aot_linear_search + .invoke_dynamic(linear_search_function_id, &[], Some(u64::MAX)) + .unwrap(); + let value = result.return_value; + assert!(matches!(value, Value::Enum { tag: 0, .. })) + }); + }); + + linear_search_group.bench_function("VM", |b| { + b.iter(|| { + let result = linear_search_runner + .run_function_with_starknet_context( + linear_search_function, + &[], + Some(usize::MAX), + StarknetState::default(), + ) + .unwrap(); + let value = result.value; + assert!(matches!(value, RunResultValue::Success(_))) + }); + }); + + linear_search_group.finish(); + } + + { + let mut factorial_group = c.benchmark_group("factorial_2M"); + + factorial_group.bench_function("Cached JIT", |b| { + b.iter(|| { + let result = jit_factorial + .invoke_dynamic(factorial_function_id, &[], Some(u64::MAX)) + .unwrap(); + let value = result.return_value; + assert!(matches!(value, Value::Enum { tag: 0, .. })) + }); + }); + factorial_group.bench_function("Cached AOT", |b| { + b.iter(|| { + let result = aot_factorial + .invoke_dynamic(factorial_function_id, &[], Some(u64::MAX)) + .unwrap(); + let value = result.return_value; + assert!(matches!(value, Value::Enum { tag: 0, .. })) + }); + }); + + factorial_group.bench_function("VM", |b| { + b.iter(|| { + let result = factorial_runner + .run_function_with_starknet_context( + factorial_function, + &[], + Some(usize::MAX), + StarknetState::default(), + ) + .unwrap(); + let value = result.value; + assert!(matches!(value, RunResultValue::Success(_))) + }); + }); + + factorial_group.finish(); + } + + { + let mut fibonacci_group = c.benchmark_group("fibonacci_2M"); + + fibonacci_group.bench_function("Cached JIT", |b| { + b.iter(|| { + let result = jit_fibonacci + .invoke_dynamic(fibonacci_function_id, &[], Some(u64::MAX)) + .unwrap(); + let value = result.return_value; + assert!(matches!(value, Value::Enum { tag: 0, .. })) + }); + }); + fibonacci_group.bench_function("Cached AOT", |b| { + b.iter(|| { + let result = aot_fibonacci + .invoke_dynamic(fibonacci_function_id, &[], Some(u64::MAX)) + .unwrap(); + let value = result.return_value; + assert!(matches!(value, Value::Enum { tag: 0, .. })) + }) + }); + fibonacci_group.bench_function("VM", |b| { + b.iter(|| { + let result = fibonacci_runner + .run_function_with_starknet_context( + fibonacci_function, + &[], + Some(usize::MAX), + StarknetState::default(), + ) + .unwrap(); + let value = result.value; + assert!(matches!(value, RunResultValue::Success(_))) + }); + }); + + fibonacci_group.finish(); + } - c.bench_function("Cached JIT factorial_2M", |b| { - b.iter(|| jit_factorial.invoke_dynamic(factorial_function_id, &[], Some(u64::MAX))); - }); - c.bench_function("Cached JIT fib_2M", |b| { - b.iter(|| jit_fibonacci.invoke_dynamic(fibonacci_function_id, &[], Some(u64::MAX))); - }); - c.bench_function("Cached JIT logistic_map", |b| { - b.iter(|| jit_logistic_map.invoke_dynamic(logistic_map_function_id, &[], Some(u64::MAX))); - }); - - c.bench_function("Cached AOT factorial_2M", |b| { - b.iter(|| aot_factorial.invoke_dynamic(factorial_function_id, &[], Some(u64::MAX))); - }); - c.bench_function("Cached AOT fib_2M", |b| { - b.iter(|| aot_fibonacci.invoke_dynamic(fibonacci_function_id, &[], Some(u64::MAX))); - }); - c.bench_function("Cached AOT logistic_map", |b| { - b.iter(|| aot_logistic_map.invoke_dynamic(logistic_map_function_id, &[], Some(u64::MAX))); - }); + { + let mut logistic_map_group = c.benchmark_group("logistic_map"); + + logistic_map_group.bench_function("Cached JIT", |b| { + b.iter(|| { + let result = jit_logistic_map + .invoke_dynamic(logistic_map_function_id, &[], Some(u64::MAX)) + .unwrap(); + let value = result.return_value; + assert!(matches!(value, Value::Enum { tag: 0, .. })) + }); + }); + + logistic_map_group.bench_function("Cached AOT", |b| { + b.iter(|| { + let result = aot_logistic_map + .invoke_dynamic(logistic_map_function_id, &[], Some(u64::MAX)) + .unwrap(); + let value = result.return_value; + assert!(matches!(value, Value::Enum { tag: 0, .. })) + }); + }); + + logistic_map_group.bench_function("VM", |b| { + b.iter(|| { + let result = logistic_map_runner + .run_function_with_starknet_context( + logistic_map_function, + &[], + Some(usize::MAX), + StarknetState::default(), + ) + .unwrap(); + let value = result.value; + assert!(matches!(value, RunResultValue::Success(_))) + }); + }); + + logistic_map_group.finish(); + } } fn load_contract(path: impl AsRef) -> Program { @@ -84,5 +255,35 @@ fn load_contract(path: impl AsRef) -> Program { sirrra_program.program } +fn load_contract_for_vm(path: impl AsRef) -> SierraCasmRunner { + let mut db = RootDatabase::builder() + .detect_corelib() + .build() + .expect("failed to build database"); + let main_crate_ids = setup_project(&mut db, path.as_ref()).expect("failed to setup project"); + let program = compile_prepared_db( + &db, + main_crate_ids.clone(), + CompilerConfig { + replace_ids: true, + ..Default::default() + }, + ) + .expect("failed to compile program"); + + let replacer = DebugReplacer { db: &db }; + let contracts = find_contracts((db).upcast(), &main_crate_ids); + let contracts_info = + get_contracts_info(&db, contracts, &replacer).expect("failed to get contracts info"); + + SierraCasmRunner::new( + program.program.clone(), + Some(Default::default()), + contracts_info, + None, + ) + .expect("failed to create runner") +} + criterion_group!(benches, criterion_benchmark); criterion_main!(benches); diff --git a/programs/benches/linear_search.c b/programs/benches/linear_search.c new file mode 100644 index 000000000..7a81f3b40 --- /dev/null +++ b/programs/benches/linear_search.c @@ -0,0 +1,37 @@ +#include +#include + + +typedef struct linear_search_return_values +{ + uint64_t range_check_counter; + uint64_t remaining_gas; + struct { + uint8_t discriminant; + struct { + void *ptr; + uint32_t start; + uint32_t end; + uint32_t cap; + } err; + } result; +} linear_search_return_values_t; + +static void run_bench(linear_search_return_values_t *, uint64_t) + __attribute__((weakref("_mlir_ciface_linear_search::linear_search::main(f4)"))); + +extern uint64_t* cairo_native__set_costs_builtin(uint64_t*); + +int main() +{ + uint64_t BuiltinCosts[7] = {1, 4050, 583, 4085, 491, 230, 604}; + + cairo_native__set_costs_builtin(&BuiltinCosts[0]); + + linear_search_return_values_t return_values; + + run_bench(&return_values, 0xFFFFFFFFFFFFFFFF); + assert((return_values.result.discriminant & 0x1) == 0); + + return 0; +} diff --git a/programs/benches/linear_search.cairo b/programs/benches/linear_search.cairo new file mode 100644 index 000000000..1cef6fb6a --- /dev/null +++ b/programs/benches/linear_search.cairo @@ -0,0 +1,37 @@ +fn search(array: @Array, number: felt252) -> u32 { + let mut index = 0; + + while index < array.len() { + if *array[index] == number { + break; + } + + index += 1; + }; + + return index; +} + +fn init_array(length: u32) -> Array { + let mut array = ArrayTrait::new(); + for i in 0..length { + array.append(i.into()); + }; + + return array; +} + +fn main() { + let array = init_array(4001); + + let index = search(@array, 4000); + assert( + index == 4000, + 'invalid result' + ); + let index = search(@array, 2000); + assert( + index == 2000, + 'invalid result' + ); +}