Only keep track of recently used stacks in memory.

.circleci/config.yml

@@ -503,7 +503,7 @@ jobs:
   ledger-with-rocksdb:
     docker:
       - image: cimg/rust:1.81.0 # Attention - Change the MSRV in Cargo.toml and rust-toolchain as well
-    resource_class: << pipeline.parameters.medium >>
+    resource_class: << pipeline.parameters.large >>
     steps:
       - run_serial:
           flags: --features=rocks
@@ -692,7 +692,7 @@ jobs:
     resource_class: << pipeline.parameters.twoxlarge >>
     steps:
       - run_serial:
-          flags: -- --ignored test_deployment_synthesis_overload test_deep_nested_execution_cost
+          flags: -- --ignored test_deployment_synthesis_overload test_deep_nested_execution_cost -- --test-threads=1
           workspace_member: synthesizer
           cache_key: v1.0.0-rust-1.81.0-snarkvm-synthesizer-mem-heavy-cache
 

console/network/src/lib.rs

@@ -203,6 +203,10 @@ pub trait Network:
     //  Increasing this value will require a migration to prevent forking during network upgrades.
     const MAX_CERTIFICATES: u16;
 
+    /// The maximum number of stacks in the process.
+    /// Must be at least Self::MAX_PROGRAM_DEPTH * Self::MAX_IMPORTS to be safe.
+    const MAX_STACKS: usize = Self::MAX_PROGRAM_DEPTH * Self::MAX_IMPORTS * 10;
+
     /// The maximum number of bytes in a transaction.
     // Note: This value must **not** be decreased as it would invalidate existing transactions.
     const MAX_TRANSACTION_SIZE: usize = 128_000; // 128 kB

ledger/block/src/lib.rs

@@ -656,7 +656,7 @@ pub mod test_helpers {
         let inputs = [address.to_string(), format!("{amount}_u64")];
 
         // Initialize the process.
-        let process = Process::load().unwrap();
+        let process = Process::load_testing_only().unwrap();
         // Authorize the function.
         let authorization =
             process.authorize::<CurrentAleo, _>(&private_key, locator.0, locator.1, inputs.iter(), rng).unwrap();

ledger/block/src/transaction/deployment/mod.rs

@@ -198,7 +198,7 @@ function compute:
                 assert!(string.is_empty(), "Parser did not consume all of the string: '{string}'");
 
                 // Construct the process.
-                let process = Process::load().unwrap();
+                let process = Process::load_testing_only().unwrap();
                 // Compute the deployment.
                 let deployment = process.deploy::<CurrentAleo, _>(&program, rng).unwrap();
                 // Return the deployment.

ledger/block/src/transaction/fee/mod.rs

@@ -241,7 +241,7 @@ pub mod test_helpers {
         let priority_fee_in_microcredits = 1_000;
 
         // Initialize the process.
-        let process = Process::load().unwrap();
+        let process = Process::load_testing_only().unwrap();
         // Authorize the fee.
         let authorization = process
             .authorize_fee_private::<CurrentAleo, _>(
@@ -298,7 +298,7 @@ pub mod test_helpers {
         let priority_fee = 1_000;
 
         // Initialize the process.
-        let process = Process::load().unwrap();
+        let process = Process::load_testing_only().unwrap();
         // Authorize the fee.
         let authorization = process
             .authorize_fee_public::<CurrentAleo, _>(

ledger/narwhal/batch-header/src/lib.rs

@@ -72,7 +72,7 @@ impl<N: Network> BatchHeader<N> {
     pub const MAX_CERTIFICATES_BEFORE_V3: u16 = 100;
     /// The maximum number of rounds to store before garbage collecting.
     pub const MAX_GC_ROUNDS: usize = 100;
-    /// The maximum number of transmissions in a batch.
+    /// The maximum number of transmissions per batch.
     /// Note: This limit is set to 50 as part of safety measures to prevent DoS attacks.
     /// This limit can be increased in the future as performance improves. Alternatively,
     /// the rate of block production can be sped up to compensate for the limit set here.

ledger/puzzle/epoch/src/synthesis/program/mod.rs

@@ -104,7 +104,7 @@ function synthesize:
         let program = Program::from_str(&program_string)?;
 
         // Initialize a new process.
-        let process = Process::<N>::load()?;
+        let process = Process::<N>::load_no_storage()?;
         // Initialize the stack with the synthesis challenge program.
         let stack = Stack::new(&process, &program)?;
 

ledger/query/src/query.rs

@@ -140,7 +140,7 @@ impl<N: Network, B: BlockStorage<N>> QueryTrait<N> for Query<N, B> {
         }
     }
 
-    /// Returns a state path for the given `commitment`.
+    /// Returns the current block height.
     fn current_block_height(&self) -> Result<u32> {
         match self {
             Self::VM(block_store) => Ok(block_store.max_height().unwrap_or_default()),
@@ -159,7 +159,7 @@ impl<N: Network, B: BlockStorage<N>> QueryTrait<N> for Query<N, B> {
         }
     }
 
-    /// Returns a state path for the given `commitment`.
+    /// Returns the current block height.
     #[cfg(feature = "async")]
     async fn current_block_height_async(&self) -> Result<u32> {
         match self {

ledger/test-helpers/src/lib.rs

@@ -150,7 +150,7 @@ function compute:
             assert!(string.is_empty(), "Parser did not consume all of the string: '{string}'");
 
             // Construct the process.
-            let process = Process::load().unwrap();
+            let process = Process::load_testing_only().unwrap();
             // Compute the deployment.
             let deployment = process.deploy::<CurrentAleo, _>(&program, rng).unwrap();
             // Return the deployment.
@@ -230,7 +230,7 @@ pub fn sample_fee_private(deployment_or_execution_id: Field<CurrentNetwork>, rng
     let priority_fee_in_microcredits = 1_000;
 
     // Initialize the process.
-    let process = Process::load().unwrap();
+    let process = Process::load_testing_only().unwrap();
     // Authorize the fee.
     let authorization = process
         .authorize_fee_private::<CurrentAleo, _>(
@@ -284,7 +284,7 @@ pub fn sample_fee_public(deployment_or_execution_id: Field<CurrentNetwork>, rng:
     let priority_fee_in_microcredits = 1_000;
 
     // Initialize the process.
-    let process = Process::load().unwrap();
+    let process = Process::load_testing_only().unwrap();
     // Authorize the fee.
     let authorization = process
         .authorize_fee_public::<CurrentAleo, _>(
@@ -399,7 +399,7 @@ pub fn sample_large_execution_transaction(rng: &mut TestRng) -> Transaction<Curr
             let program = large_transaction_program();
 
             // Construct the process.
-            let mut process = synthesizer_process::Process::load().unwrap();
+            let mut process = synthesizer_process::Process::load_testing_only().unwrap();
             // Add the program.
             process.add_program(&program).unwrap();
 
@@ -510,7 +510,7 @@ fn sample_genesis_block_and_components_raw(
     let inputs = [address.to_string(), format!("{amount}_u64")];
 
     // Initialize the process.
-    let process = Process::load().unwrap();
+    let process = Process::load_testing_only().unwrap();
     // Authorize the function.
     let authorization =
         process.authorize::<CurrentAleo, _>(&private_key, locator.0, locator.1, inputs.iter(), rng).unwrap();

synthesizer/Cargo.toml

@@ -32,7 +32,7 @@ aleo-cli = [ ]
 async = [ "ledger-query/async", "synthesizer-process/async" ]
 cuda = [ "algorithms/cuda" ]
 history = [ "serde" ]
-rocks = [ "ledger-store/rocks" ]
+rocks = ["ledger-store/rocks", "synthesizer-process/rocks"]
 serial = [
   "console/serial",
   "ledger-block/serial",

synthesizer/process/Cargo.toml

@@ -95,7 +95,7 @@ version = "=1.2.1"
 
 [dependencies.aleo-std]
 version = "0.1.24"
-default-features = false
+features = ["storage"]
 
 [dependencies.colored]
 version = "2"
@@ -104,6 +104,9 @@ version = "2"
 version = "2.0"
 features = [ "serde" ]
 
+[dependencies.lru]
+version = "0.12"
+
 [dependencies.once_cell]
 version = "1.18"
 
@@ -124,6 +127,9 @@ optional = true
 version = "1.0"
 features = [ "preserve_order" ]
 
+[dependencies.tracing]
+version = "0.1"
+
 [dev-dependencies.bincode]
 version = "1.3"
 

synthesizer/process/benches/stack_operations.rs

@@ -41,7 +41,7 @@ fn bench_stack_new(c: &mut Criterion) {
     let mut rng = TestRng::default();
 
     // Initialize a process.
-    let mut process = Process::load().unwrap();
+    let mut process = Process::load_testing_only().unwrap();
 
     // Benchmark the base case.
     c.bench_function("Depth 0 | Stack::new", |b| {
@@ -98,7 +98,7 @@ fn bench_stack_get_number_of_calls(c: &mut Criterion) {
     const DEPTHS: [usize; 6] = [1, 2, 4, 8, 16, 30];
 
     // Initialize a process.
-    let mut process = Process::load().unwrap();
+    let mut process = Process::load_testing_only().unwrap();
 
     // Add the 0th program to the process.
     add_program_at_depth(&mut process, 0);

synthesizer/process/src/cost.rs

@@ -87,7 +87,7 @@ pub fn execution_cost_v1<N: Network>(process: &Process<N>, execution: &Execution
 
     // Get the finalize cost for the root transition.
     let stack = process.get_stack(transition.program_id())?;
-    let finalize_cost = cost_in_microcredits_v1(stack, transition.function_name())?;
+    let finalize_cost = cost_in_microcredits_v1(&stack, transition.function_name())?;
 
     // Compute the total cost in microcredits.
     let total_cost = storage_cost
@@ -522,7 +522,7 @@ function over_five_thousand:
         let threshold = MainnetV0::EXECUTION_STORAGE_PENALTY_THRESHOLD;
 
         // Test the cost of an execution.
-        let mut process = Process::load().unwrap();
+        let mut process = Process::load_testing_only().unwrap();
 
         // Get the program.
         let program = Program::from_str(SIZE_BOUNDARY_PROGRAM).unwrap();

synthesizer/process/src/deploy.rs

@@ -38,10 +38,39 @@ impl<N: Network> Process<N> {
         deployment
     }
 
-    /// Adds the newly-deployed program.
+    /// Loads the stack and imported stacks for the given program ID into memory.
+    #[inline]
+    pub fn load_stack(&self, program_id: impl TryInto<ProgramID<N>>) -> Result<Arc<Stack<N>>> {
+        let program_id = program_id.try_into().map_err(|_| anyhow!("Invalid program ID"))?;
+        debug!("Lazy loading stack for {program_id}");
+        // Retrieve the stores.
+        let store = self.store.as_ref().ok_or_else(|| anyhow!("Failed to get store"))?;
+        // Retrieve the transaction store.
+        let transaction_store = store.transaction_store();
+
+        // Retrieve the deployment store.
+        let deployment_store = transaction_store.deployment_store();
+        // Retrieve the transaction ID.
+        let transaction_id = deployment_store
+            .find_transaction_id_from_program_id(&program_id)
+            .map_err(|e| anyhow!("Program ID not found in storage: {e}"))?
+            .ok_or_else(|| anyhow!("Program ID not found in storage"))?;
+
+        // Retrieve the deployment from the transaction ID.
+        let deployment = match transaction_store.get_deployment(&transaction_id)? {
+            Some(deployment) => deployment,
+            None => bail!("Deployment transaction '{transaction_id}' is not found in storage."),
+        };
+
+        // Load the deployment into memory and return it.
+        // When initializing the corresponding Stack, each import Stack will be loaded recursively.
+        self.load_deployment(deployment)
+    }
+
+    /// Constructs, loads and returns the Stack from the deployment.
     /// This method assumes the given deployment **is valid**.
     #[inline]
-    pub fn load_deployment(&mut self, deployment: &Deployment<N>) -> Result<()> {
+    fn load_deployment(&self, deployment: Deployment<N>) -> Result<Arc<Stack<N>>> {
         let timer = timer!("Process::load_deployment");
 
         // Compute the program stack.
@@ -54,11 +83,14 @@ impl<N: Network> Process<N> {
         }
         lap!(timer, "Insert the verifying keys");
 
+        // Wrap the stack in an Arc.
+        let stack = Arc::new(stack);
+
         // Add the stack to the process.
-        self.add_stack(stack);
+        self.add_stack(stack.clone())?;
 
         finish!(timer);
 
-        Ok(())
+        Ok(stack)
     }
 }

synthesizer/process/src/execute.rs

@@ -72,7 +72,7 @@ mod tests {
         let rng = &mut TestRng::default();
 
         // Initialize the process.
-        let process = Process::<CurrentNetwork>::load().unwrap();
+        let process = Process::<CurrentNetwork>::load_testing_only().unwrap();
 
         // Sample a private key.
         let private_key = PrivateKey::<CurrentNetwork>::new(rng).unwrap();
@@ -124,7 +124,7 @@ mod tests {
         let rng = &mut TestRng::default();
 
         // Initialize the process.
-        let process = Process::<CurrentNetwork>::load().unwrap();
+        let process = Process::<CurrentNetwork>::load_testing_only().unwrap();
 
         // Sample a private key.
         let private_key = PrivateKey::new(rng).unwrap();

synthesizer/process/src/finalize.rs

@@ -54,7 +54,7 @@ impl<N: Network> Process<N> {
             // Retrieve the fee stack.
             let fee_stack = self.get_stack(fee.program_id())?;
             // Finalize the fee transition.
-            finalize_operations.extend(finalize_fee_transition(state, store, fee_stack, fee)?);
+            finalize_operations.extend(finalize_fee_transition(state, store, &fee_stack, fee)?);
             lap!(timer, "Finalize transition for '{}/{}'", fee.program_id(), fee.function_name());
 
             /* Finalize the deployment. */
@@ -114,15 +114,15 @@ impl<N: Network> Process<N> {
             // Finalize the root transition.
             // Note that this will result in all the remaining transitions being finalized, since the number
             // of calls matches the number of transitions.
-            let mut finalize_operations = finalize_transition(state, store, stack, transition, call_graph)?;
+            let mut finalize_operations = finalize_transition(state, store, &stack, transition, call_graph)?;
 
             /* Finalize the fee. */
 
             if let Some(fee) = fee {
                 // Retrieve the fee stack.
                 let fee_stack = self.get_stack(fee.program_id())?;
                 // Finalize the fee transition.
-                finalize_operations.extend(finalize_fee_transition(state, store, fee_stack, fee)?);
+                finalize_operations.extend(finalize_fee_transition(state, store, &fee_stack, fee)?);
                 lap!(timer, "Finalize transition for '{}/{}'", fee.program_id(), fee.function_name());
             }
 
@@ -148,7 +148,7 @@ impl<N: Network> Process<N> {
             // Retrieve the stack.
             let stack = self.get_stack(fee.program_id())?;
             // Finalize the fee transition.
-            let result = finalize_fee_transition(state, store, stack, fee);
+            let result = finalize_fee_transition(state, store, &stack, fee);
             finish!(timer, "Finalize transition for '{}/{}'", fee.program_id(), fee.function_name());
             // Return the result.
             result
@@ -520,7 +520,7 @@ function compute:
         .unwrap();
 
         // Initialize a new process.
-        let mut process = Process::load().unwrap();
+        let process = Process::load_testing_only().unwrap();
         // Deploy the program.
         let deployment = process.deploy::<CurrentAleo, _>(&program, rng).unwrap();
 
@@ -538,7 +538,7 @@ function compute:
         let (stack, _) =
             process.finalize_deployment(sample_finalize_state(1), &finalize_store, &deployment, &fee).unwrap();
         // Add the stack *manually* to the process.
-        process.add_stack(stack);
+        process.add_stack(Arc::new(stack)).unwrap();
 
         // Ensure the program exists.
         assert!(process.contains_program(program.id()));