Criterion Benchmark Libfuncs and more (#385)

* bench libfuncs

* bench cairo vm too

* trigger

* trigger

* allow unused which is used

* fix unique id issue

* fix box and nullable from_jit

* fixes

* clippy

Cargo.toml

@@ -71,6 +71,7 @@ pretty_assertions_sorted = "1.2.3"
 proptest = "1.2"
 tempfile = "3.6"
 test-case = "3.2.1"
+walkdir = "2"
 
 [build-dependencies]
 cc = "1.0"
@@ -95,5 +96,9 @@ opt-level = 1
 name = "compile_time"
 harness = false
 
+[[bench]]
+name = "libfuncs"
+harness = false
+
 [workspace]
 members = ["runtime"]

benches/compile_time.rs

@@ -5,12 +5,12 @@ use util::prepare_programs;
 mod util;
 
 pub fn bench_compile_time(c: &mut Criterion) {
+    let programs = prepare_programs("programs/compile_benches");
+
     {
         let mut c = c.benchmark_group("Compilation With Independent Context");
 
-        let programs = prepare_programs();
-
-        for (program, filename) in programs {
+        for (program, filename) in &programs {
             c.bench_with_input(BenchmarkId::new(filename, 1), &program, |b, program| {
                 b.iter(|| {
                     let native_context = NativeContext::new();
@@ -24,11 +24,9 @@ pub fn bench_compile_time(c: &mut Criterion) {
     {
         let mut c = c.benchmark_group("Compilation With Shared Context");
 
-        let programs = prepare_programs();
-
         let native_context = NativeContext::new();
 
-        for (program, filename) in programs {
+        for (program, filename) in &programs {
             c.bench_with_input(BenchmarkId::new(filename, 1), &program, |b, program| {
                 b.iter(|| {
                     native_context.compile(program).unwrap();
@@ -41,9 +39,7 @@ pub fn bench_compile_time(c: &mut Criterion) {
     {
         let mut c = c.benchmark_group("Compilation With Independent Context To Object Code");
 
-        let programs = prepare_programs();
-
-        for (program, filename) in programs {
+        for (program, filename) in &programs {
             c.bench_with_input(BenchmarkId::new(filename, 1), &program, |b, program| {
                 b.iter(|| {
                     let native_context = NativeContext::new();
@@ -59,11 +55,9 @@ pub fn bench_compile_time(c: &mut Criterion) {
     {
         let mut c = c.benchmark_group("Compilation With Shared Context To Object Code");
 
-        let programs = prepare_programs();
-
         let native_context = NativeContext::new();
 
-        for (program, filename) in programs {
+        for (program, filename) in &programs {
             c.bench_with_input(BenchmarkId::new(filename, 1), &program, |b, program| {
                 b.iter(|| {
                     let module = native_context.compile(black_box(program)).unwrap();

benches/libfuncs.rs

@@ -0,0 +1,100 @@
+use cairo_lang_runner::StarknetState;
+use cairo_native::{context::NativeContext, executor::NativeExecutor};
+use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
+use util::{create_vm_runner, prepare_programs};
+
+mod util;
+
+pub fn bench_libfuncs(c: &mut Criterion) {
+    let programs = prepare_programs("tests/cases");
+
+    {
+        let mut c = c.benchmark_group("Libfunc Execution Time");
+
+        for (program, filename) in &programs {
+            if filename == "tests/cases/felt_ops/div.cairo" {
+                continue; // todo: enable when libfuncs felt252_div and felt252_div_const are implemented
+            }
+
+            let entry = program
+                .funcs
+                .iter()
+                .find(|f| {
+                    if let Some(name) = &f.id.debug_name {
+                        name.ends_with("main")
+                    } else {
+                        false
+                    }
+                })
+                .expect("failed to find entry point");
+
+            let vm_runner = create_vm_runner(program);
+
+            c.bench_with_input(
+                BenchmarkId::new(filename, "SierraCasmRunner"),
+                &program,
+                |b, _program| {
+                    b.iter(|| {
+                        let res = vm_runner
+                            .run_function_with_starknet_context(
+                                entry,
+                                &[],
+                                Some(usize::MAX),
+                                StarknetState::default(),
+                            )
+                            .expect("should run correctly");
+                        black_box(res)
+                    })
+                },
+            );
+
+            c.bench_with_input(
+                BenchmarkId::new(filename, "jit-cold"),
+                &program,
+                |b, program| {
+                    let native_context = NativeContext::new();
+                    b.iter(|| {
+                        let module = native_context.compile(program).unwrap();
+                        // pass manager internally verifies the MLIR output is correct.
+                        let native_executor = NativeExecutor::new(module);
+
+                        // Execute the program.
+                        let result = native_executor
+                            .execute(&entry.id, &[], Some(u64::MAX as u128))
+                            .unwrap();
+                        black_box(result)
+                    })
+                },
+            );
+
+            c.bench_with_input(
+                BenchmarkId::new(filename, "jit-hot"),
+                program,
+                |b, program| {
+                    let native_context = NativeContext::new();
+                    let module = native_context.compile(program).unwrap();
+                    // pass manager internally verifies the MLIR output is correct.
+                    let native_executor = NativeExecutor::new(module);
+
+                    // warmup
+                    for _ in 0..5 {
+                        native_executor
+                            .execute(&entry.id, &[], Some(u64::MAX as u128))
+                            .unwrap();
+                    }
+
+                    b.iter(|| {
+                        // Execute the program.
+                        let result = native_executor
+                            .execute(&entry.id, &[], Some(u64::MAX as u128))
+                            .unwrap();
+                        black_box(result)
+                    })
+                },
+            );
+        }
+    }
+}
+
+criterion_group!(benches, bench_libfuncs);
+criterion_main!(benches);

benches/util.rs

@@ -1,22 +1,32 @@
-use std::{path::Path, sync::Arc};
+use std::sync::Arc;
 
+use cairo_lang_runner::SierraCasmRunner;
 use cairo_lang_sierra::program::Program;
+use walkdir::WalkDir;
 
-pub fn prepare_programs() -> impl Iterator<Item = (Arc<Program>, String)> {
-    let programs = Path::new("programs/compile_benches")
-        .read_dir()
-        .unwrap()
+pub fn prepare_programs(path: &str) -> Vec<(Arc<Program>, String)> {
+    WalkDir::new(path)
+        .into_iter()
         .filter_map(|entry| {
-            let path = entry.unwrap().path();
+            let e = entry.unwrap();
+            let path = e.path();
             match path.extension().map(|x| x.to_str().unwrap()) {
                 Some("cairo") => Some((
-                    cairo_native::utils::cairo_to_sierra(&path),
-                    path.file_name().unwrap().to_str().unwrap().to_string(),
+                    cairo_native::utils::cairo_to_sierra(path),
+                    path.display().to_string(),
                 )),
                 _ => None,
             }
         })
-        .collect::<Vec<_>>(); // collect so iter is not lazy evaluated on bench
+        .collect::<Vec<_>>()
+}
 
-    programs.into_iter()
+#[allow(unused)] // its used but clippy doesn't detect it well
+pub fn create_vm_runner(program: &Program) -> SierraCasmRunner {
+    SierraCasmRunner::new(
+        program.clone(),
+        Some(Default::default()),
+        Default::default(),
+    )
+    .unwrap()
 }

src/libfuncs/box.rs

@@ -1,4 +1,6 @@
 //! # Box libfuncs
+//!
+//! A heap allocated value, its internally a pointer that can't be null.
 
 use super::{LibfuncBuilder, LibfuncHelper};
 use crate::{
@@ -166,4 +168,20 @@ mod test {
 
         run_program_assert_output(&program, "run_test", &[], &[JITValue::Uint32(2)]);
     }
+
+    #[test]
+    fn run_box() {
+        let program = load_cairo!(
+            use box::BoxTrait;
+            use box::BoxImpl;
+
+            fn run_test() -> Box<u32>  {
+                let x: u32 = 2_u32;
+                let box_x: Box<u32> = BoxTrait::new(x);
+                box_x
+            }
+        );
+
+        run_program_assert_output(&program, "run_test", &[], &[JITValue::Uint32(2)]);
+    }
 }

src/libfuncs/gas.rs

@@ -49,7 +49,9 @@ where
             build_withdraw_gas(context, registry, entry, location, helper, metadata, info)
         }
         GasConcreteLibfunc::RedepositGas(_) => todo!(),
-        GasConcreteLibfunc::GetAvailableGas(_) => todo!(),
+        GasConcreteLibfunc::GetAvailableGas(info) => {
+            build_get_available_gas(context, registry, entry, location, helper, metadata, info)
+        }
         GasConcreteLibfunc::BuiltinWithdrawGas(info) => {
             build_builtin_withdraw_gas(context, registry, entry, location, helper, metadata, info)
         }
@@ -59,6 +61,30 @@ where
     }
 }
 
+/// Generate MLIR operations for the `get_builtin_costs` libfunc.
+pub fn build_get_available_gas<'ctx, 'this, TType, TLibfunc>(
+    _context: &'ctx Context,
+    _registry: &ProgramRegistry<TType, TLibfunc>,
+    entry: &'this Block<'ctx>,
+    location: Location<'ctx>,
+    helper: &LibfuncHelper<'ctx, 'this>,
+    _metadata: &mut MetadataStorage,
+    _info: &SignatureOnlyConcreteLibfunc,
+) -> Result<()>
+where
+    TType: GenericType,
+    TLibfunc: GenericLibfunc,
+    <TType as GenericType>::Concrete: TypeBuilder<TType, TLibfunc, Error = CoreTypeBuilderError>,
+    <TLibfunc as GenericLibfunc>::Concrete: LibfuncBuilder<TType, TLibfunc, Error = Error>,
+{
+    entry.append_operation(helper.br(
+        0,
+        &[entry.argument(0)?.into(), entry.argument(0)?.into()],
+        location,
+    ));
+    Ok(())
+}
+
 /// Generate MLIR operations for the `withdraw_gas` libfunc.
 pub fn build_withdraw_gas<'ctx, 'this, TType, TLibfunc>(
     context: &'ctx Context,

src/libfuncs/nullable.rs

@@ -1,4 +1,6 @@
 //! # Nullable libfuncs
+//!
+//! Like a Box but it can be null.
 
 use super::{LibfuncBuilder, LibfuncHelper};
 use crate::{
@@ -170,7 +172,10 @@ where
 
 #[cfg(test)]
 mod test {
-    use crate::utils::test::{jit_struct, load_cairo, run_program_assert_output};
+    use crate::{
+        utils::test::{jit_struct, load_cairo, run_program_assert_output},
+        values::JITValue,
+    };
 
     #[test]
     fn run_null() {
@@ -189,6 +194,24 @@ mod test {
         run_program_assert_output(&program, "run_test", &[], &[jit_struct!()]);
     }
 
+    #[test]
+    fn run_null_jit() {
+        let program = load_cairo!(
+            use nullable::null;
+            use nullable::match_nullable;
+            use nullable::FromNullableResult;
+            use nullable::nullable_from_box;
+            use box::BoxTrait;
+
+            fn run_test() -> Nullable<u8> {
+                let a: Nullable<u8> = null();
+                a
+            }
+        );
+
+        run_program_assert_output(&program, "run_test", &[], &[JITValue::Null]);
+    }
+
     #[test]
     fn run_not_null() {
         let program = load_cairo!(