examples/finetune_pretrained.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LLaMA with FSDP and Tensor Parallelism\n",
    "In this notebook we provide a simple exmple of using scalax sjit for combining FSDP and tensor parallelism (TP). We train a simple LLaMA transformer model defined in the huggingface transformers library using huggingface datasets. We show that all of these can be done rather easily without modifying the model definition and the tran step function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "h9HtQYyDAI0H"
   },
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "\n",
    "import jax\n",
    "import jax.numpy as jnp\n",
    "from flax.training.train_state import TrainState\n",
    "import optax\n",
    "\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "from tqdm.notebook import tqdm, trange\n",
    "\n",
    "from transformers import FlaxLlamaForCausalLM, LlamaTokenizer\n",
    "from datasets import load_dataset\n",
    "\n",
    "from scalax.utils import JaxRNG\n",
    "from scalax.sharding import PartitionSpec, MeshShardingHelper, TreePathShardingRule\n",
    "from scalax.utils import print_pytree_structure\n",
    "\n",
    "from typing import Callable\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## C4 DataLoader\n",
    "We use the C4 dataset implemented in the Huggingface datasets library. We create a very simple data loader that tokenizes the text and apply sequnce packing to form batches."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def c4_data_loader(batch_size, seq_length):\n",
    "    chunk_size = batch_size * seq_length\n",
    "    # Stream the c4 dataset without downloading\n",
    "    dataset = load_dataset('allenai/c4', 'en', split='train', streaming=True, trust_remote_code=True)\n",
    "    # Use the free OpenLLaMA tokenizer\n",
    "    tokenizer = LlamaTokenizer.from_pretrained('h2oai/h2o-danube3-500m-chat')\n",
    "\n",
    "    # Use a static mask and position ids here for simplicity\n",
    "    attention_mask = np.ones((batch_size, seq_length), dtype=np.int32)\n",
    "    position_ids = np.arange(seq_length, dtype=np.int32).reshape(1, -1)\n",
    "    position_ids = np.broadcast_to(position_ids, (batch_size, seq_length))\n",
    "    \n",
    "    token_buffer = []\n",
    "    for example in dataset:\n",
    "        # Sequence packing\n",
    "        token_buffer.extend(tokenizer.encode(example['text']))\n",
    "        token_buffer.append(tokenizer.eos_token_id)\n",
    "        while len(token_buffer) > chunk_size + 1:\n",
    "            batch = {\n",
    "                'input_tokens': np.array(\n",
    "                    token_buffer[:chunk_size], dtype=np.int32\n",
    "                ).reshape(batch_size, seq_length),  # Input tokens to language model\n",
    "                'target_tokens': np.array(\n",
    "                    token_buffer[1:chunk_size + 1], dtype=np.int32\n",
    "                ).reshape(batch_size, seq_length),  # Shift-by-one training target tokens\n",
    "                'attention_mask': attention_mask,\n",
    "                'position_ids': position_ids,\n",
    "            }\n",
    "            token_buffer = token_buffer[chunk_size:]\n",
    "            yield batch        \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inspect the Model Tree Sturcture\n",
    "To compose FSDP and tensor parallelism, we need to specify which axis should be sharded with fsdp and which axis should be sharded with tp for each tensor. To do this, we need to use `TreePathShardingRule`. `TreePathShardingRule` specifies sharding according to the path of leaf tensors in a pytree, and match it agaist user defined regular expressions. To use it, the fisrt step is to inspect the pytree structure of the model. We can do this with `scalax.utils.print_pytree_structure`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some of the weights of FlaxLlamaForCausalLM were initialized in bfloat16 precision from the model checkpoint at h2oai/h2o-danube3-500m-chat:\n",
      "[('lm_head', 'kernel'), ('model', 'embed_tokens', 'embedding'), ('model', 'layers', '0', 'input_layernorm', 'weight'), ('model', 'layers', '0', 'mlp', 'down_proj', 'kernel'), ('model', 'layers', '0', 'mlp', 'gate_proj', 'kernel'), ('model', 'layers', '0', 'mlp', 'up_proj', 'kernel'), ('model', 'layers', '0', 'post_attention_layernorm', 'weight'), ('model', 'layers', '0', 'self_attn', 'k_proj', 'kernel'), ('model', 'layers', '0', 'self_attn', 'o_proj', 'kernel'), ('model', 'layers', '0', 'self_attn', 'q_proj', 'kernel'), ('model', 'layers', '0', 'self_attn', 'v_proj', 'kernel'), ('model', 'layers', '1', 'input_layernorm', 'weight'), ('model', 'layers', '1', 'mlp', 'down_proj', 'kernel'), ('model', 'layers', '1', 'mlp', 'gate_proj', 'kernel'), ('model', 'layers', '1', 'mlp', 'up_proj', 'kernel'), ('model', 'layers', '1', 'post_attention_layernorm', 'weight'), ('model', 'layers', '1', 'self_attn', 'k_proj', 'kernel'), ('model', 'layers', '1', 'self_attn', 'o_proj', 'kernel'), ('model', 'layers', '1', 'self_attn', 'q_proj', 'kernel'), ('model', 'layers', '1', 'self_attn', 'v_proj', 'kernel'), ('model', 'layers', '10', 'input_layernorm', 'weight'), ('model', 'layers', '10', 'mlp', 'down_proj', 'kernel'), ('model', 'layers', '10', 'mlp', 'gate_proj', 'kernel'), ('model', 'layers', '10', 'mlp', 'up_proj', 'kernel'), ('model', 'layers', '10', 'post_attention_layernorm', 'weight'), ('model', 'layers', '10', 'self_attn', 'k_proj', 'kernel'), ('model', 'layers', '10', 'self_attn', 'o_proj', 'kernel'), ('model', 'layers', '10', 'self_attn', 'q_proj', 'kernel'), ('model', 'layers', '10', 'self_attn', 'v_proj', 'kernel'), ('model', 'layers', '11', 'input_layernorm', 'weight'), ('model', 'layers', '11', 'mlp', 'down_proj', 'kernel'), ('model', 'layers', '11', 'mlp', 'gate_proj', 'kernel'), ('model', 'layers', '11', 'mlp', 'up_proj', 'kernel'), ('model', 'layers', '11', 'post_attention_layernorm', 'weight'), ('model', 'layers', '11', 'self_attn', 'k_proj', 'kernel'), ('model', 'layers', '11', 'self_attn', 'o_proj', 'kernel'), ('model', 'layers', '11', 'self_attn', 'q_proj', 'kernel'), ('model', 'layers', '11', 'self_attn', 'v_proj', 'kernel'), ('model', 'layers', '12', 'input_layernorm', 'weight'), ('model', 'layers', '12', 'mlp', 'down_proj', 'kernel'), ('model', 'layers', '12', 'mlp', 'gate_proj', 'kernel'), ('model', 'layers', '12', 'mlp', 'up_proj', 'kernel'), ('model', 'layers', '12', 'post_attention_layernorm', 'weight'), ('model', 'layers', '12', 'self_attn', 'k_proj', 'kernel'), ('model', 'layers', '12', 'self_attn', 'o_proj', 'kernel'), ('model', 'layers', '12', 'self_attn', 'q_proj', 'kernel'), ('model', 'layers', '12', 'self_attn', 'v_proj', 'kernel'), ('model', 'layers', '13', 'input_layernorm', 'weight'), ('model', 'layers', '13', 'mlp', 'down_proj', 'kernel'), ('model', 'layers', '13', 'mlp', 'gate_proj', 'kernel'), ('model', 'layers', '13', 'mlp', 'up_proj', 'kernel'), ('model', 'layers', '13', 'post_attention_layernorm', 'weight'), ('model', 'layers', '13', 'self_attn', 'k_proj', 'kernel'), ('model', 'layers', '13', 'self_attn', 'o_proj', 'kernel'), ('model', 'layers', '13', 'self_attn', 'q_proj', 'kernel'), ('model', 'layers', '13', 'self_attn', 'v_proj', 'kernel'), ('model', 'layers', '14', 'input_layernorm', 'weight'), ('model', 'layers', '14', 'mlp', 'down_proj', 'kernel'), ('model', 'layers', '14', 'mlp', 'gate_proj', 'kernel'), ('model', 'layers', '14', 'mlp', 'up_proj', 'kernel'), ('model', 'layers', '14', 'post_attention_layernorm', 'weight'), ('model', 'layers', '14', 'self_attn', 'k_proj', 'kernel'), ('model', 'layers', '14', 'self_attn', 'o_proj', 'kernel'), ('model', 'layers', '14', 'self_attn', 'q_proj', 'kernel'), ('model', 'layers', '14', 'self_attn', 'v_proj', 'kernel'), ('model', 'layers', '15', 'input_layernorm', 'weight'), ('model', 'layers', '15', 'mlp', 'down_proj', 'kernel'), ('model', 'layers', '15', 'mlp', 'gate_proj', 'kernel'), ('model', 'layers', '15', 'mlp', 'up_proj', 'kernel'), ('model', 'layers', '15', 'post_attention_layernorm', 'weight'), ('model', 'layers', '15', 'self_attn', 'k_proj', 'kernel'), ('model', 'layers', '15', 'self_attn', 'o_proj', 'kernel'), ('model', 'layers', '15', 'self_attn', 'q_proj', 'kernel'), ('model', 'layers', '15', 'self_attn', 'v_proj', 'kernel'), ('model', 'layers', '2', 'input_layernorm', 'weight'), ('model', 'layers', '2', 'mlp', 'down_proj', 'kernel'), ('model', 'layers', '2', 'mlp', 'gate_proj', 'kernel'), ('model', 'layers', '2', 'mlp', 'up_proj', 'kernel'), ('model', 'layers', '2', 'post_attention_layernorm', 'weight'), ('model', 'layers', '2', 'self_attn', 'k_proj', 'kernel'), ('model', 'layers', '2', 'self_attn', 'o_proj', 'kernel'), ('model', 'layers', '2', 'self_attn', 'q_proj', 'kernel'), ('model', 'layers', '2', 'self_attn', 'v_proj', 'kernel'), ('model', 'layers', '3', 'input_layernorm', 'weight'), ('model', 'layers', '3', 'mlp', 'down_proj', 'kernel'), ('model', 'layers', '3', 'mlp', 'gate_proj', 'kernel'), ('model', 'layers', '3', 'mlp', 'up_proj', 'kernel'), ('model', 'layers', '3', 'post_attention_layernorm', 'weight'), ('model', 'layers', '3', 'self_attn', 'k_proj', 'kernel'), ('model', 'layers', '3', 'self_attn', 'o_proj', 'kernel'), ('model', 'layers', '3', 'self_attn', 'q_proj', 'kernel'), ('model', 'layers', '3', 'self_attn', 'v_proj', 'kernel'), ('model', 'layers', '4', 'input_layernorm', 'weight'), ('model', 'layers', '4', 'mlp', 'down_proj', 'kernel'), ('model', 'layers', '4', 'mlp', 'gate_proj', 'kernel'), ('model', 'layers', '4', 'mlp', 'up_proj', 'kernel'), ('model', 'layers', '4', 'post_attention_layernorm', 'weight'), ('model', 'layers', '4', 'self_attn', 'k_proj', 'kernel'), ('model', 'layers', '4', 'self_attn', 'o_proj', 'kernel'), ('model', 'layers', '4', 'self_attn', 'q_proj', 'kernel'), ('model', 'layers', '4', 'self_attn', 'v_proj', 'kernel'), ('model', 'layers', '5', 'input_layernorm', 'weight'), ('model', 'layers', '5', 'mlp', 'down_proj', 'kernel'), ('model', 'layers', '5', 'mlp', 'gate_proj', 'kernel'), ('model', 'layers', '5', 'mlp', 'up_proj', 'kernel'), ('model', 'layers', '5', 'post_attention_layernorm', 'weight'), ('model', 'layers', '5', 'self_attn', 'k_proj', 'kernel'), ('model', 'layers', '5', 'self_attn', 'o_proj', 'kernel'), ('model', 'layers', '5', 'self_attn', 'q_proj', 'kernel'), ('model', 'layers', '5', 'self_attn', 'v_proj', 'kernel'), ('model', 'layers', '6', 'input_layernorm', 'weight'), ('model', 'layers', '6', 'mlp', 'down_proj', 'kernel'), ('model', 'layers', '6', 'mlp', 'gate_proj', 'kernel'), ('model', 'layers', '6', 'mlp', 'up_proj', 'kernel'), ('model', 'layers', '6', 'post_attention_layernorm', 'weight'), ('model', 'layers', '6', 'self_attn', 'k_proj', 'kernel'), ('model', 'layers', '6', 'self_attn', 'o_proj', 'kernel'), ('model', 'layers', '6', 'self_attn', 'q_proj', 'kernel'), ('model', 'layers', '6', 'self_attn', 'v_proj', 'kernel'), ('model', 'layers', '7', 'input_layernorm', 'weight'), ('model', 'layers', '7', 'mlp', 'down_proj', 'kernel'), ('model', 'layers', '7', 'mlp', 'gate_proj', 'kernel'), ('model', 'layers', '7', 'mlp', 'up_proj', 'kernel'), ('model', 'layers', '7', 'post_attention_layernorm', 'weight'), ('model', 'layers', '7', 'self_attn', 'k_proj', 'kernel'), ('model', 'layers', '7', 'self_attn', 'o_proj', 'kernel'), ('model', 'layers', '7', 'self_attn', 'q_proj', 'kernel'), ('model', 'layers', '7', 'self_attn', 'v_proj', 'kernel'), ('model', 'layers', '8', 'input_layernorm', 'weight'), ('model', 'layers', '8', 'mlp', 'down_proj', 'kernel'), ('model', 'layers', '8', 'mlp', 'gate_proj', 'kernel'), ('model', 'layers', '8', 'mlp', 'up_proj', 'kernel'), ('model', 'layers', '8', 'post_attention_layernorm', 'weight'), ('model', 'layers', '8', 'self_attn', 'k_proj', 'kernel'), ('model', 'layers', '8', 'self_attn', 'o_proj', 'kernel'), ('model', 'layers', '8', 'self_attn', 'q_proj', 'kernel'), ('model', 'layers', '8', 'self_attn', 'v_proj', 'kernel'), ('model', 'layers', '9', 'input_layernorm', 'weight'), ('model', 'layers', '9', 'mlp', 'down_proj', 'kernel'), ('model', 'layers', '9', 'mlp', 'gate_proj', 'kernel'), ('model', 'layers', '9', 'mlp', 'up_proj', 'kernel'), ('model', 'layers', '9', 'post_attention_layernorm', 'weight'), ('model', 'layers', '9', 'self_attn', 'k_proj', 'kernel'), ('model', 'layers', '9', 'self_attn', 'o_proj', 'kernel'), ('model', 'layers', '9', 'self_attn', 'q_proj', 'kernel'), ('model', 'layers', '9', 'self_attn', 'v_proj', 'kernel'), ('model', 'norm', 'weight')]\n",
      "You should probably UPCAST the model weights to float32 if this was not intended. See [`~FlaxPreTrainedModel.to_fp32`] for further information on how to do this.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lm_head/kernel: shape: (1536, 32000), dtype: bfloat16\n",
      "model/embed_tokens/embedding: shape: (32000, 1536), dtype: bfloat16\n",
      "model/layers/0/input_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/0/mlp/down_proj/kernel: shape: (4096, 1536), dtype: bfloat16\n",
      "model/layers/0/mlp/gate_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/0/mlp/up_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/0/post_attention_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/0/self_attn/k_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/0/self_attn/o_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/0/self_attn/q_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/0/self_attn/v_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/1/input_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/1/mlp/down_proj/kernel: shape: (4096, 1536), dtype: bfloat16\n",
      "model/layers/1/mlp/gate_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/1/mlp/up_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/1/post_attention_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/1/self_attn/k_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/1/self_attn/o_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/1/self_attn/q_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/1/self_attn/v_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/10/input_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/10/mlp/down_proj/kernel: shape: (4096, 1536), dtype: bfloat16\n",
      "model/layers/10/mlp/gate_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/10/mlp/up_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/10/post_attention_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/10/self_attn/k_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/10/self_attn/o_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/10/self_attn/q_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/10/self_attn/v_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/11/input_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/11/mlp/down_proj/kernel: shape: (4096, 1536), dtype: bfloat16\n",
      "model/layers/11/mlp/gate_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/11/mlp/up_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/11/post_attention_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/11/self_attn/k_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/11/self_attn/o_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/11/self_attn/q_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/11/self_attn/v_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/12/input_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/12/mlp/down_proj/kernel: shape: (4096, 1536), dtype: bfloat16\n",
      "model/layers/12/mlp/gate_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/12/mlp/up_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/12/post_attention_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/12/self_attn/k_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/12/self_attn/o_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/12/self_attn/q_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/12/self_attn/v_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/13/input_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/13/mlp/down_proj/kernel: shape: (4096, 1536), dtype: bfloat16\n",
      "model/layers/13/mlp/gate_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/13/mlp/up_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/13/post_attention_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/13/self_attn/k_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/13/self_attn/o_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/13/self_attn/q_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/13/self_attn/v_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/14/input_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/14/mlp/down_proj/kernel: shape: (4096, 1536), dtype: bfloat16\n",
      "model/layers/14/mlp/gate_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/14/mlp/up_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/14/post_attention_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/14/self_attn/k_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/14/self_attn/o_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/14/self_attn/q_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/14/self_attn/v_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/15/input_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/15/mlp/down_proj/kernel: shape: (4096, 1536), dtype: bfloat16\n",
      "model/layers/15/mlp/gate_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/15/mlp/up_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/15/post_attention_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/15/self_attn/k_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/15/self_attn/o_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/15/self_attn/q_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/15/self_attn/v_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/2/input_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/2/mlp/down_proj/kernel: shape: (4096, 1536), dtype: bfloat16\n",
      "model/layers/2/mlp/gate_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/2/mlp/up_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/2/post_attention_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/2/self_attn/k_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/2/self_attn/o_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/2/self_attn/q_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/2/self_attn/v_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/3/input_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/3/mlp/down_proj/kernel: shape: (4096, 1536), dtype: bfloat16\n",
      "model/layers/3/mlp/gate_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/3/mlp/up_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/3/post_attention_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/3/self_attn/k_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/3/self_attn/o_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/3/self_attn/q_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/3/self_attn/v_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/4/input_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/4/mlp/down_proj/kernel: shape: (4096, 1536), dtype: bfloat16\n",
      "model/layers/4/mlp/gate_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/4/mlp/up_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/4/post_attention_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/4/self_attn/k_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/4/self_attn/o_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/4/self_attn/q_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/4/self_attn/v_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/5/input_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/5/mlp/down_proj/kernel: shape: (4096, 1536), dtype: bfloat16\n",
      "model/layers/5/mlp/gate_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/5/mlp/up_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/5/post_attention_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/5/self_attn/k_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/5/self_attn/o_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/5/self_attn/q_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/5/self_attn/v_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/6/input_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/6/mlp/down_proj/kernel: shape: (4096, 1536), dtype: bfloat16\n",
      "model/layers/6/mlp/gate_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/6/mlp/up_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/6/post_attention_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/6/self_attn/k_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/6/self_attn/o_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/6/self_attn/q_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/6/self_attn/v_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/7/input_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/7/mlp/down_proj/kernel: shape: (4096, 1536), dtype: bfloat16\n",
      "model/layers/7/mlp/gate_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/7/mlp/up_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/7/post_attention_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/7/self_attn/k_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/7/self_attn/o_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/7/self_attn/q_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/7/self_attn/v_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/8/input_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/8/mlp/down_proj/kernel: shape: (4096, 1536), dtype: bfloat16\n",
      "model/layers/8/mlp/gate_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/8/mlp/up_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/8/post_attention_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/8/self_attn/k_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/8/self_attn/o_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/8/self_attn/q_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/8/self_attn/v_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/9/input_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/9/mlp/down_proj/kernel: shape: (4096, 1536), dtype: bfloat16\n",
      "model/layers/9/mlp/gate_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/9/mlp/up_proj/kernel: shape: (1536, 4096), dtype: bfloat16\n",
      "model/layers/9/post_attention_layernorm/weight: shape: (1536,), dtype: bfloat16\n",
      "model/layers/9/self_attn/k_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/layers/9/self_attn/o_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/9/self_attn/q_proj/kernel: shape: (1536, 1536), dtype: bfloat16\n",
      "model/layers/9/self_attn/v_proj/kernel: shape: (1536, 768), dtype: bfloat16\n",
      "model/norm/weight: shape: (1536,), dtype: bfloat16\n"
     ]
    }
   ],
   "source": [
    "def load_model() -> tuple[Callable, dict]:\n",
    "    with jax.default_device(jax.devices('cpu')[0]):\n",
    "        model = FlaxLlamaForCausalLM.from_pretrained('h2oai/h2o-danube3-500m-chat')\n",
    "        forward = model.__call__\n",
    "        params = model.params\n",
    "    return forward, params\n",
    "\n",
    "\n",
    "forward, params = load_model()\n",
    "\n",
    "print_pytree_structure(params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define the Mesh Sharding Rule\n",
    "According to the previous pytree structure, we define the following mesh and sharding rule. We employ 2-way tensor parallelism and use the rest for FSDP."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'lm_head': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       " 'model': {'embed_tokens': {'embedding': PartitionSpec('fsdp', 'tp')},\n",
       "  'layers': {'0': {'input_layernorm': {'weight': PartitionSpec()},\n",
       "    'mlp': {'down_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'gate_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'up_proj': {'kernel': PartitionSpec('fsdp', 'tp')}},\n",
       "    'post_attention_layernorm': {'weight': PartitionSpec()},\n",
       "    'self_attn': {'k_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'o_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'q_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'v_proj': {'kernel': PartitionSpec('fsdp', 'tp')}}},\n",
       "   '1': {'input_layernorm': {'weight': PartitionSpec()},\n",
       "    'mlp': {'down_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'gate_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'up_proj': {'kernel': PartitionSpec('fsdp', 'tp')}},\n",
       "    'post_attention_layernorm': {'weight': PartitionSpec()},\n",
       "    'self_attn': {'k_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'o_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'q_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'v_proj': {'kernel': PartitionSpec('fsdp', 'tp')}}},\n",
       "   '10': {'input_layernorm': {'weight': PartitionSpec()},\n",
       "    'mlp': {'down_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'gate_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'up_proj': {'kernel': PartitionSpec('fsdp', 'tp')}},\n",
       "    'post_attention_layernorm': {'weight': PartitionSpec()},\n",
       "    'self_attn': {'k_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'o_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'q_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'v_proj': {'kernel': PartitionSpec('fsdp', 'tp')}}},\n",
       "   '11': {'input_layernorm': {'weight': PartitionSpec()},\n",
       "    'mlp': {'down_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'gate_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'up_proj': {'kernel': PartitionSpec('fsdp', 'tp')}},\n",
       "    'post_attention_layernorm': {'weight': PartitionSpec()},\n",
       "    'self_attn': {'k_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'o_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'q_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'v_proj': {'kernel': PartitionSpec('fsdp', 'tp')}}},\n",
       "   '12': {'input_layernorm': {'weight': PartitionSpec()},\n",
       "    'mlp': {'down_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'gate_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'up_proj': {'kernel': PartitionSpec('fsdp', 'tp')}},\n",
       "    'post_attention_layernorm': {'weight': PartitionSpec()},\n",
       "    'self_attn': {'k_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'o_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'q_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'v_proj': {'kernel': PartitionSpec('fsdp', 'tp')}}},\n",
       "   '13': {'input_layernorm': {'weight': PartitionSpec()},\n",
       "    'mlp': {'down_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'gate_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'up_proj': {'kernel': PartitionSpec('fsdp', 'tp')}},\n",
       "    'post_attention_layernorm': {'weight': PartitionSpec()},\n",
       "    'self_attn': {'k_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'o_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'q_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'v_proj': {'kernel': PartitionSpec('fsdp', 'tp')}}},\n",
       "   '14': {'input_layernorm': {'weight': PartitionSpec()},\n",
       "    'mlp': {'down_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'gate_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'up_proj': {'kernel': PartitionSpec('fsdp', 'tp')}},\n",
       "    'post_attention_layernorm': {'weight': PartitionSpec()},\n",
       "    'self_attn': {'k_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'o_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'q_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'v_proj': {'kernel': PartitionSpec('fsdp', 'tp')}}},\n",
       "   '15': {'input_layernorm': {'weight': PartitionSpec()},\n",
       "    'mlp': {'down_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'gate_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'up_proj': {'kernel': PartitionSpec('fsdp', 'tp')}},\n",
       "    'post_attention_layernorm': {'weight': PartitionSpec()},\n",
       "    'self_attn': {'k_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'o_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'q_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'v_proj': {'kernel': PartitionSpec('fsdp', 'tp')}}},\n",
       "   '2': {'input_layernorm': {'weight': PartitionSpec()},\n",
       "    'mlp': {'down_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'gate_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'up_proj': {'kernel': PartitionSpec('fsdp', 'tp')}},\n",
       "    'post_attention_layernorm': {'weight': PartitionSpec()},\n",
       "    'self_attn': {'k_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'o_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'q_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'v_proj': {'kernel': PartitionSpec('fsdp', 'tp')}}},\n",
       "   '3': {'input_layernorm': {'weight': PartitionSpec()},\n",
       "    'mlp': {'down_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'gate_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'up_proj': {'kernel': PartitionSpec('fsdp', 'tp')}},\n",
       "    'post_attention_layernorm': {'weight': PartitionSpec()},\n",
       "    'self_attn': {'k_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'o_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'q_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'v_proj': {'kernel': PartitionSpec('fsdp', 'tp')}}},\n",
       "   '4': {'input_layernorm': {'weight': PartitionSpec()},\n",
       "    'mlp': {'down_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'gate_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'up_proj': {'kernel': PartitionSpec('fsdp', 'tp')}},\n",
       "    'post_attention_layernorm': {'weight': PartitionSpec()},\n",
       "    'self_attn': {'k_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'o_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'q_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'v_proj': {'kernel': PartitionSpec('fsdp', 'tp')}}},\n",
       "   '5': {'input_layernorm': {'weight': PartitionSpec()},\n",
       "    'mlp': {'down_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'gate_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'up_proj': {'kernel': PartitionSpec('fsdp', 'tp')}},\n",
       "    'post_attention_layernorm': {'weight': PartitionSpec()},\n",
       "    'self_attn': {'k_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'o_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'q_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'v_proj': {'kernel': PartitionSpec('fsdp', 'tp')}}},\n",
       "   '6': {'input_layernorm': {'weight': PartitionSpec()},\n",
       "    'mlp': {'down_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'gate_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'up_proj': {'kernel': PartitionSpec('fsdp', 'tp')}},\n",
       "    'post_attention_layernorm': {'weight': PartitionSpec()},\n",
       "    'self_attn': {'k_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'o_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'q_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'v_proj': {'kernel': PartitionSpec('fsdp', 'tp')}}},\n",
       "   '7': {'input_layernorm': {'weight': PartitionSpec()},\n",
       "    'mlp': {'down_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'gate_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'up_proj': {'kernel': PartitionSpec('fsdp', 'tp')}},\n",
       "    'post_attention_layernorm': {'weight': PartitionSpec()},\n",
       "    'self_attn': {'k_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'o_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'q_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'v_proj': {'kernel': PartitionSpec('fsdp', 'tp')}}},\n",
       "   '8': {'input_layernorm': {'weight': PartitionSpec()},\n",
       "    'mlp': {'down_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'gate_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'up_proj': {'kernel': PartitionSpec('fsdp', 'tp')}},\n",
       "    'post_attention_layernorm': {'weight': PartitionSpec()},\n",
       "    'self_attn': {'k_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'o_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'q_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'v_proj': {'kernel': PartitionSpec('fsdp', 'tp')}}},\n",
       "   '9': {'input_layernorm': {'weight': PartitionSpec()},\n",
       "    'mlp': {'down_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'gate_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'up_proj': {'kernel': PartitionSpec('fsdp', 'tp')}},\n",
       "    'post_attention_layernorm': {'weight': PartitionSpec()},\n",
       "    'self_attn': {'k_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'o_proj': {'kernel': PartitionSpec('tp', 'fsdp')},\n",
       "     'q_proj': {'kernel': PartitionSpec('fsdp', 'tp')},\n",
       "     'v_proj': {'kernel': PartitionSpec('fsdp', 'tp')}}}},\n",
       "  'norm': {'weight': PartitionSpec()}}}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mesh = MeshShardingHelper([-1, 2], ['fsdp', 'tp'])\n",
    "\n",
    "# Use pairs of regex, PartitionSpec to specify the sharding\n",
    "model_sharding_rule = TreePathShardingRule(\n",
    "    ('embedding', PartitionSpec('fsdp', 'tp')),\n",
    "    ('lm_head/kernel', PartitionSpec('tp', 'fsdp')),\n",
    "    # Megatron style feedfoward sharding\n",
    "    ('mlp/(up|gate)_proj/kernel', PartitionSpec('fsdp', 'tp')),\n",
    "    ('mlp/down_proj/kernel', PartitionSpec('tp', 'fsdp')),\n",
    "    # Attention should be sharded by heads\n",
    "    ('self_attn/(k|q|v)_proj/kernel', PartitionSpec('fsdp', 'tp')),\n",
    "    ('self_attn/o_proj/kernel', PartitionSpec('tp', 'fsdp')),\n",
    "    ('norm', PartitionSpec()),\n",
    ")\n",
    "\n",
    "# Inspect the sharding rule can be successfully applied\n",
    "model_sharding_rule.apply(params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sharded Initialization and TrainStep\n",
    "We define the initialization and train step functions and shard them with the `sjit` method of `MeshShardingHelper`. For model, we use our previously defined sharding rule, and for data, we shard the batch axis with `fsdp`. All of these can be simply specified as arugments to `sjit`, without changing the `init_fn` and `train_step` functions. The `init_fn` and `train_step` functions here are exactly the same as if we are training on a single GPU."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "8aiua0iqAa5A"
   },
   "outputs": [],
   "source": [
    "optimizer = optax.adamw(1e-5)\n",
    "seq_length = 512\n",
    "\n",
    "@partial(\n",
    "    mesh.sjit,  # Shard the initialization function\n",
    "    # The initialized model and optimizer states should be sharded according\n",
    "    # to our tree path sharding rule\n",
    "    out_shardings=model_sharding_rule,\n",
    ")\n",
    "def init_fn():\n",
    "    return TrainState.create(params=params, tx=optimizer, apply_fn=None)\n",
    "\n",
    "\n",
    "@partial(\n",
    "    mesh.sjit,  # Shard the train step function\n",
    "    # The input model should be sharded with model_sharding_rule, the data is loaded as replicated\n",
    "    in_shardings=(model_sharding_rule, PartitionSpec()),\n",
    "    # The output model should be sharded with model_sharding_rule, the loss metrics is replicated\n",
    "    out_shardings=(model_sharding_rule, PartitionSpec()),\n",
    "    # After the beginning of the function, the data should be sharded along the `fsdp` axis\n",
    "    args_sharding_constraint=(model_sharding_rule, PartitionSpec('fsdp')),\n",
    "    donate_argnums=(0,),  # No need to preserve the old model\n",
    ")\n",
    "def train_step(train_state, batch):\n",
    "    def loss_fn(params):\n",
    "        logits = forward(\n",
    "            input_ids=batch['input_tokens'],\n",
    "            attention_mask=batch['attention_mask'],\n",
    "            position_ids=batch['position_ids'],\n",
    "            params=params,\n",
    "        ).logits\n",
    "        loss = optax.softmax_cross_entropy_with_integer_labels(\n",
    "            logits, batch['target_tokens']\n",
    "        )\n",
    "        return jnp.mean(loss)\n",
    "    grad_fn = jax.value_and_grad(loss_fn)\n",
    "    loss, grads = grad_fn(train_state.params)\n",
    "    train_state = train_state.apply_gradients(grads=grads)\n",
    "    return train_state, loss\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running the Sharded Train Step\n",
    "Now let's run the sharded train step for 100 steps and plot the loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "f1lJqS7afXo5"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8e20880a03c8414392c3cec6506e60d6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "35b7e59ec1b24a528ec27d10e61631d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Resolving data files:   0%|          | 0/1024 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b161de8687544d2bbc9cb64696d2c55",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Resolving data files:   0%|          | 0/1024 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7edcec159670>]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGdCAYAAAAxCSikAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAACS1UlEQVR4nO29eZwU9Z3//6q+e+6BYS7OQRBEwAOVjHitEsEYlRx+o5uIJhp/cWFXvtmNWbJqEhMXY5JNdDerMfHYfBXJmqhJiIoGBS9AQVHwQJAbZoZz7umz6vdH9+dTn66u6q7qrp6ubt7Px2MeSk91T01N96fen9f79X6/JUVRFBAEQRAEQTgYV7FPgCAIgiAIIhsUsBAEQRAE4XgoYCEIgiAIwvFQwEIQBEEQhOOhgIUgCIIgCMdDAQtBEARBEI6HAhaCIAiCIBwPBSwEQRAEQTgeT7FPwA5kWcbBgwdRXV0NSZKKfToEQRAEQZhAURT09fWhtbUVLldmDaUsApaDBw9i7NixxT4NgiAIgiByYN++fRgzZkzGY8oiYKmurgaQ+IVramqKfDYEQRAEQZiht7cXY8eO5ffxTJRFwMLSQDU1NRSwEARBEESJYcbOQaZbgiAIgiAcDwUsBEEQBEE4HgpYCIIgCIJwPBSwEARBEATheChgIQiCIAjC8VDAQhAEQRCE46GAhSAIgiAIx0MBC0EQBEEQjocCFoIgCIIgHA8FLARBEARBOB4KWAiCIAiCcDwUsBAEQRAE4XgoYCEIomyIywr+3/o9+Lizt9inQhCEzVDAQhBE2bB8wx7c8exW3P3Xj4p9KgRB2AwFLARBlAWyrODRN3YDAI4NRIp7MgRB2A4FLARBlAVrPzmMnUcGAADhmFzksyEIwm4oYCEIoix45I1d/P/DsXgRz4QgiEJAAQtBECXP9q4+vLb9CP93hBQWgig7KGAhCKLkefTN3QCAyY1VACglRBDlCAUsBEGUNN2DETz9zn4AwP934UkAgHCUAhaCKDcoYCEIoqR58q19CEVlTGupwQWTGwAkPCyKohT5zAiCsBMKWAiCKFmicRm/W7cbAPD1ORPg97oBALICxGQKWAiinPAU+wQIgiByZdUHnejoCaGhyocrTmtN+V44JsPrpj0ZQZQL9GkmCKJkeXz9HgDA388ej4DXDZ8QoFClEEGUFxSwEARRsuw9OggAuHhqIwDA5ZJ40EK9WAiivKCAhSCIkiUST6gofo+6lLH/p0ohgigvKGAhCKJkYf1WfGLA4nWlfI8giPLAUsDywAMPYObMmaipqUFNTQ3a29vx/PPPGx7/2GOPQZKklK9AIJByjKIouPPOO9HS0oJgMIi5c+di+/btuf02BEGcUDCfiuhd8XsSlUKUEiKI8sJSwDJmzBjcc8892LRpEzZu3IiLL74YV111FT744APD59TU1KCjo4N/7dmzJ+X79957L+6//348+OCD2LBhAyorKzFv3jyEQqHcfiOCIE4IFEXJnBIihYUgygpLZc1XXHFFyr/vvvtuPPDAA1i/fj1OPfVU3edIkoTm5mbd7ymKgl/+8pe4/fbbcdVVVwEAfve736GpqQnPPvssrrnmGiunRxDECURMVsB6w4kpIR95WAiiLMnZwxKPx7FixQoMDAygvb3d8Lj+/n6MHz8eY8eOTVNjdu3ahc7OTsydO5c/Vltbi9mzZ2PdunWGrxkOh9Hb25vyRRDEiYVYtuzTUVgicUoJEUQ5YTlg2bJlC6qqquD3+/Gtb30LzzzzDKZNm6Z77JQpU/DII4/gT3/6Ex5//HHIsoxzzz0X+/cn5n50dnYCAJqamlKe19TUxL+nx7Jly1BbW8u/xo4da/XXIAiixEkJWPQ8LKSwEERZYTlgmTJlCjZv3owNGzbglltuwfXXX48PP/xQ99j29nYsXLgQp59+Oi688EI8/fTTGDVqFH7961/nddJLly5FT08P/9q3b19er0cQROnB/CsuCfC4qUqIIMody635fT4fJk2aBACYNWsW3n77bdx3332mghCv14szzjgDO3bsAADubenq6kJLSws/rqurC6effrrh6/j9fvj9fqunThBEGRHRKWkGRNMtpYQIopzIuw+LLMsIh8Omjo3H49iyZQsPTtra2tDc3IzVq1fzY3p7e7Fhw4aMvhiCIIiwTkkzIJY1k8JCEOWEJYVl6dKluOyyyzBu3Dj09fVh+fLlWLNmDVatWgUAWLhwIUaPHo1ly5YBAO666y585jOfwaRJk9Dd3Y2f/vSn2LNnD2666SYAiQqiJUuW4Mc//jEmT56MtrY23HHHHWhtbcWCBQvs/U0JgigrVIXFnfI4dboliPLEUsBy6NAhLFy4EB0dHaitrcXMmTOxatUqfPaznwUA7N27Fy6Xuts5fvw4vvnNb6KzsxP19fWYNWsW3nzzzRST7m233YaBgQHcfPPN6O7uxnnnnYcXXnghrcEcQRCEiF4PFkAoa6aUEEGUFZKisE4GpUtvby9qa2vR09ODmpqaYp8OQRDDwFu7juH//Hod2hoq8cq/XMQf//6ftuJ/1u3BP108Cd++dErxTpAgiKxYuX/TLCGCIEoSvbb8AOD3koeFIMoRClgIgihJWGM44yohClgIopyggIUgiJKEypoJ4sSCAhaCIEqSrGXNVCVEEGUFBSwEQZQkRgqLj1JCBFGWUMBCEERJwsqaycNCECcGFLAQBFGSGHpYvORhIYhyhAIWgiBKEhawaBvHUWt+gihPKGAhCKIkMQ5YKCVEEOUIBSwEQZQk3MNiWCVEKSGCKCcoYCEIoiTJViUUIYWFIMoKClgIgihJwlkbx1HAQhDlBAUsBEGUJGpKyJ3yuFolRAELQZQTFLAQBFGSGLfmZ1VC5GEhiHKCAhaCIEqS7LOESGEhiHKCAhaCIEqSbAFLJCZDUZRhPy+CIAoDBSwEQZQkzMPi15Y1e1VPC6ksBFE+UMBCEERJYljWLAQwFLAQRPlAAQtBECWJUcDidUuQpMT/k/GWIMoHClgIgihJwgadbiVJSvGxEARRHlDAQhBESWKksAA0AJEgyhEKWAiCKEkiyXSPfsCSLG2OUsBCEOUCBSwEQZQkvNOtXsDCu92Sh4UgygUKWAiCKEl4SsidvoyxxyglRBDlAwUsBEGUJCxg8ZOHhSBOCChgIQiiJMlouvVSlRBBlBsUsBAEUZJk9LB4yMNCEOUGBSwEQZQcsqwgGk/MCdLzsPCUEFUJEUTZQAELQRAlB1NXgGwKCwUsBFEuUMBCEETJkS1g8VFKiCDKDgpYCIIoOUQzbcaUECksBFE2UMBCEETJIfZgkdikQwHeOI48LARRNlDAQhBEyZGppBlQPSyROKWECKJcoICFIIiSI1NJM0BVQgRRjlDAQhBEyZGpLT9AVUIEUY5QwEIQRMkRzpYSouGHBFF2UMBCEETJkc3DQsMPCaL8oICFIIiSg3tYjFJCXvKwEES5QQELQRAlh9kqIUoJEUT5QAELQRAlh/myZlJYCKJcsBSwPPDAA5g5cyZqampQU1OD9vZ2PP/884bH/+Y3v8H555+P+vp61NfXY+7cuXjrrbdSjrnhhhsgSVLK1/z583P7bQiCOCFg/VX8VNZMECcMlgKWMWPG4J577sGmTZuwceNGXHzxxbjqqqvwwQcf6B6/Zs0aXHvttXjllVewbt06jB07FpdeeikOHDiQctz8+fPR0dHBv5588sncfyOCIMqerGXNXjLdEkS54bFy8BVXXJHy77vvvhsPPPAA1q9fj1NPPTXt+CeeeCLl37/97W/xxz/+EatXr8bChQv5436/H83NzVZOhSCIE5isKSE3eVgIotzI2cMSj8exYsUKDAwMoL293dRzBgcHEY1GMWLEiJTH16xZg8bGRkyZMgW33HILjh49mvF1wuEwent7U74IgjhxMN+HhRQWgigXLCksALBlyxa0t7cjFAqhqqoKzzzzDKZNm2bqud/97nfR2tqKuXPn8sfmz5+PL37xi2hra8Onn36K733ve7jsssuwbt06uN1u3ddZtmwZfvjDH1o9dYIgyoSsZc3kYSGIssNywDJlyhRs3rwZPT09+MMf/oDrr78ea9euzRq03HPPPVixYgXWrFmDQCDAH7/mmmv4/8+YMQMzZ87ESSedhDVr1uCSSy7Rfa2lS5fi29/+Nv93b28vxo4da/VXIQiiRKEqIYI48bCcEvL5fJg0aRJmzZqFZcuW4bTTTsN9992X8Tk/+9nPcM899+DFF1/EzJkzMx47ceJENDQ0YMeOHYbH+P1+XqnEvgiCOHHIHrAwhYU8LARRLlhWWLTIsoxwOGz4/XvvvRd33303Vq1ahbPOOivr6+3fvx9Hjx5FS0tLvqdGEESZkjVgIQ8LQZQdlgKWpUuX4rLLLsO4cePQ19eH5cuXY82aNVi1ahUAYOHChRg9ejSWLVsGAPjJT36CO++8E8uXL8eECRPQ2dkJAKiqqkJVVRX6+/vxwx/+EF/60pfQ3NyMTz/9FLfddhsmTZqEefPm2fyrEgRRLrBUj9/Aw8K8LTFZQSwuw2NwHEEQpYOlgOXQoUNYuHAhOjo6UFtbi5kzZ2LVqlX47Gc/CwDYu3cvXC51YXjggQcQiUTw5S9/OeV1vv/97+MHP/gB3G433n//ffzP//wPuru70draiksvvRQ/+tGP4Pf7bfj1CIIoR8wqLEAiuKGAhSBKH0sBy8MPP5zx+2vWrEn59+7duzMeHwwGuTpDEARhFrPTmoFEpVCFb1hOiyCIAkLbDoIgSo5wlrJmj9sFj0tKHEs+FoIoCyhgIQii5FAVFv1eTYBQ2kwBC0GUBRSwEARRcmRLCQGA35ssbab2/ARRFlDAQhBEyWEqYPFQaTNBlBMUsBAEUXJka80PqMEMKSwEUR5QwEIQRMnBFBa/GYXFxDyh7V19WPbcRzg+ELHnBAmCsB0KWAiCKDnMpYSYhyV7wPLg2p349as7sfL9g/acIEEQtkMBC0EQJQdPCZnysGRPCfWGogCAvnDMhrMjCKIQUMBCEETJwRWWDB4WK/OEQskhiWbSRwRBFAcKWAiCKDnCNqeEWMDClBuCIJwHBSwEQZQckWSaJ1PAwtQXcwGLnHxdClgIwqlQwFIAdhzqx8JH3sKmPceKfSoEUZaYKWvmKaFodg/LEEsJUQk0QTgWS8MPCXP89f0OvPrJYYyuC2DW+BHFPh2CKCsURbFW1mwlJUQKC0E4FlJYCgDbrQ1GaLdGEHYTkxXISuL/7fOwyKaPJQiiOFDAUgDYbi1kQoomCMIaogpipqzZjGpCCgtBOB8KWAoA26WFqESSIGwnJWAxVdacfeNAAQtBOB8KWAoAM/kNkcJCELbDDLcuCfBkmiXkNpcSisZlxJI5JkoJEYRzoYClAIRirAkVBSwEYTdm2vIDYpVQ5iBETN2SwkIQzoUClgLAFkhKCRGE/YRNdLkFzLfmFz+nVNZMEM6FApYCwBSWEC1+BGE7qsLiznic2SohUWGhlBBBOBcKWApAiCssFLAQhN0wD0umHizi960ELNSanyCcCwUsBYDJykPUh4UgbMeqhyViJSVEaVyCcCwUsBQArrCQvEwQtmNmUjNgISUUI4WFIEoBClgKAFNYIjEZMmvJSRCELUTi2Qcfit/PppqISihV9hGEc6GApQCkVh3Qjo0g7MR0Ssh0lRApLARRClDAUgDEBZCMtwRhL9bLmrOlhNTvR2IyFIVUUYJwIhSwFABxgaTSZoKwF/MKi0kPi5ASkhXwrrcEQTgLClhsRpaVlG6ZVClEEPbC0jbmq4TMm27NHE8QRHGggMVmtLs56nZLEPZSSA9L4nj6zBKEE6GAxWa0iyOlhAjCXljA4s/iYfEJHpZMvpShSGqAQgoLQTgTClhsRquokOmWIOzFqodFUYBo3DhgoZQQQZQGFLDYTJq8TCkhgrAV0x4W4fuZ0kJanxkNQCQIZ0IBi81o899DpLAQhK2Y73QrBizGGwdtgEIeFoJwJhSw2IxWYaGUEEHYS9hkSkiSpBQfixHaNC4FLAThTChgsRmqEiKIwmI2JQSoKksmX4o2JUQeFoJwJhSw2AwpLARRWMyabgFzpc1ppltqz08QjoQCFptJC1jIwEcQtmLWwwII3W4zKJ3pRnn6zBKEE6GAxWbSUkLU6ZYgbIX3YbGksGRICWmCGVJYCMKZUMBiM+kKCy1+BGEnVjwsPhMpIaaouF1S8t/0mSUIJ0IBi81oAxTysBCEvVjysHjNp4RqAp7E65PCQhCOxFLA8sADD2DmzJmoqalBTU0N2tvb8fzzz2d8zlNPPYWpU6ciEAhgxowZeO6551K+rygK7rzzTrS0tCAYDGLu3LnYvn279d/EIWjz3xSwEIS9qB4Wd9ZjzaWEEp/R2qA35fUJgnAWlgKWMWPG4J577sGmTZuwceNGXHzxxbjqqqvwwQcf6B7/5ptv4tprr8WNN96Id999FwsWLMCCBQuwdetWfsy9996L+++/Hw8++CA2bNiAyspKzJs3D6FQKL/frEhQWTNBFJZwLmXN8QxVQsnPKAtYqNMtsfVAD77+6Fv4uLO32KdCCFgKWK644gp87nOfw+TJk3HyySfj7rvvRlVVFdavX697/H333Yf58+fjO9/5Dk455RT86Ec/wplnnon/+q//ApBQV375y1/i9ttvx1VXXYWZM2fid7/7HQ4ePIhnn30271+uGDCFxetO5MOp0y1B2EtOZc1mUkKksBBJ/rBpP17ZdhhPbthb7FMhBHL2sMTjcaxYsQIDAwNob2/XPWbdunWYO3duymPz5s3DunXrAAC7du1CZ2dnyjG1tbWYPXs2P6bUYB4WtlujlBBB2EskqYBYKms2CEJkWeHfq+EKCwUsJzq9Q1EAwM4jA0U+E0LEY/UJW7ZsQXt7O0KhEKqqqvDMM89g2rRpusd2dnaiqakp5bGmpiZ0dnby77PHjI7RIxwOIxwO83/39jpHtgsJ+fAj/RGqOCAIm8ml061RmkcMTsjDQjD6wjEAwC4KWByFZYVlypQp2Lx5MzZs2IBbbrkF119/PT788MNCnJshy5YtQ21tLf8aO3bssP78TLAApb7CB4AaxxGE3Vjqw+LNnBISFdCaACksRIL+UCJgOdA9RCq5g7AcsPh8PkyaNAmzZs3CsmXLcNppp+G+++7TPba5uRldXV0pj3V1daG5uZl/nz1mdIweS5cuRU9PD//at2+f1V+jYLAAhVJCBFEYrHlYMqeEhgTPWYUv87HEiUN/UmFRFGDP0cEinw3ByLsPiyzLKekZkfb2dqxevTrlsZdeeol7Xtra2tDc3JxyTG9vLzZs2GDoiwEAv9/PS6vZl1PgKaEKFrDQ4kcQdmKtNT+rEsqssAQ8bh4AUUqIYAELAOw83F/EMyFELHlYli5dissuuwzjxo1DX18fli9fjjVr1mDVqlUAgIULF2L06NFYtmwZAODWW2/FhRdeiJ///Oe4/PLLsWLFCmzcuBEPPfQQgMT49yVLluDHP/4xJk+ejLa2Ntxxxx1obW3FggUL7P1Nhwm2O6sLJlJCVCVEEPaSU6dbg88h21AEfG5TgxKJE4O+kBCwkI/FMVgKWA4dOoSFCxeio6MDtbW1mDlzJlatWoXPfvazAIC9e/fC5VIXkXPPPRfLly/H7bffju9973uYPHkynn32WUyfPp0fc9ttt2FgYAA333wzuru7cd555+GFF15AIBCw6VccXtiOra6CUkIEYTeyrCAaVwBYNd1mTgkFvC5SWAhOfzjK/3/nYQpYnIKlgOXhhx/O+P01a9akPXb11Vfj6quvNnyOJEm46667cNddd1k5FcfCdmwsYKEqIYKwDzG1Y4eHJSykhNix1Jr/xCYal1NS+buOUErIKdAsIZsJa/qwROIy4rJSzFMiiLIhJWAx42HxZk7zMJN80OcW0kcUsJzIDAj+FYBSQk6CAhabCWvmkgCUEycIuxDTNVZMt0ZByFAk6WHxuPnrkcJyYsP8K57k9O7uwSiOD0SKeUpEEgpYbEarsADAUIQCFoKwAxaweN0SXMkbSiaypYSYx8zvdWVVYwh7+PBgL27+3UZs6+wr9qnowgKWugofWmsTXsqdlBZyBBSw2AxbACt8Hi4xh8jERwwjcVmBopRnGtJKSTMglDUbBSwsJeR1w+8m0+1w8ORbe/Hih1145t0DxT4VXVhJc3XAg4mjqgCQ8dYpUMBiM3zH5nEhwAIWqhQihom4rODy+1/D1Q+uK8ugxUpJs3ickWrC1M+A180VFgpYCktHzxAAYDASy3JkcWAVQlV+D9oaKgFQi36nYHmWEJEZJj0HvG4EvG70hmIUsJQIiqJAkrKnGZzMsYEIPk5K7YOROCr95fURt9LlFjBRJcQ/ry743NTpdjjo6AkBcG6qnKWExICFFBZnQAqLjcTiMmLJiqCA14WAN7EAUrdb53O0P4yLf74W3/795mKfSl7EZPW91jMUzXBkaRK2GrB4M/dhYZuJoJc63Q4XnSxgcehGjqWEqgIeTBxFCouToIDFRkSvit/jRsBLKaFS4X/W7cGuIwN4fqvxlPBSIBpT00DlGLDk6mEx6nSbkhLK0mSOyJ9QNI6jyYobp66LbPBhtd+DiQ0JD8uuowPUnsIBUMBiI+Ki6Pe4EOQKizM/mESCUDSOx9fvAZDY9Tk1t24GsSS3LAMW7mFxmzo+a5VQTA1YSGEpPId61blzpaCwjK4Pwud2IRKTcbB7qMhnRlDAYiMhQa52uST4KSVUEvzxnf04JvRZONJXuj0XokLA0j1YhgGLZQ9LtpSQ6jkTByWWo2HZCTDDLVAaHha3S8L4kRUAqIGcE6CAxUbCQoUQAMHD4swPJpGYTfPwa7tSHjsyoD99vBSIxdUbbW85KizJwMNvMiWUTTXRmyUEnHhpod+/vRff/cP7iBW4aV5nb4j//5BDN3KiwgJA9bHQ1OaiQwGLjYi7NQBqWTM1onIsL398CDuPDKA64MGkxkS++mh/6Sos5Z8SSnyWrCoskbgMWceDENYx3bLjTyR+8dJ2/H7jPry3v6egP4dVCAHAkENTr6KHBQDakj4WUliKDwUsNsICE63C4lTpkwB+89pOAMDfnzMO40ckpN+j/aWrsETLPWCxXCWkel30ghBxkyEaeU+0eULsvXK4L5TlyPzoFAMWhyrPauO4RLdyqhRyDhSw2EhYo7Aw0+2JJi+XClv292DDrmPwuCTcMGcCRlb5AABHSjhgEVNCZR2wWKwSAvSDEDElJEmSmkI6gRSWWFzm1+FwgdXFkvCwhFUPCwBMpF4sjoECFhtRKw5cKf8lD4szYerK52e2oKU2iIYqPwDgSAmnhMpdYbHah8XjksBGDul1uw1F1SohIHsZdDnSL0wnPtJX2GBdVFicWozQH0p2uuUelkRK6ED3EK3lRYYCFhtRTbdJD4uDTLdU9ZDKge4h/HVLBwDgpvMnAgBGJgOWoyU8mbX8PSzWAhZJkjKWNhsFLCeSwsKqYgDgcIHVRdHDEonLBTf55kK/RmGpr/DyYba7j5LKUkwoYLERsc03AMeUNSuKgmseWo+//816ClyS/PcrOxCXFbRPHInpo2sBAA0sJVTgXWYhOWFSQiYDFvFY/YAlNY3rOwEHIIoBSyHf+9G4nBYQOXEwLDfdJhUWSZKoRb9DoIDFRvhujSssictbbHNZbyiGDbuO4c1Pj+LwMNyMFUVBX8i5N8vH3tiFJzbsBQDcctFJ/PEGrrCUbsBS7ikhqx4WQOzFYpwSYn4z/wnoOxM/q4VUWA73haEogNetzutymo8lLisYSJ5TlTCHi4y3zoACFhthuzU2v4QFLsVOCYm7xX3HC9+t8ScvbMOsH/0Nb+8+VvCfZZW/vHcQP1z5IQDgnz97Mi44eRT/HjPd5lvW/ORbe3Hf37bn9Rq5UvYpIdaHxYLCkmmeUCia6js74RWWAgYsLB3UVBNwbBfwAaHUmnlYANV4+yn1YikqFLDYCNvBsUAl6LM/JdTZE8Kf3ztoKfcr7roPDEN76U17jiESl/HYG7sL/rOs8Pr2I/j2/26GogAL28dj8cWTUr4/sjKhsBwbjOQ8N+RQbwjfe2YLfvG3T4qyuGkVlnJLAVr1sABCe37N51BRFKFKiCksxmpMsdm4+xgWL38nxbhqB6mm20jB3jPsvFtqA3xtLLb6rIWlg3xuF3/fAKrxlhSW4kIBi42oCktqSsjOxe9Hf/0Q//Tku1j98SHTzxF3i/uPD9p2LkawHdtLH3ahe9AZBtatB3rw//2/jYjGFVw+owXfv+JUSJKUcsyISh8kCVAUpLTqt8JzWzrA1vt9xwp/rbWIHhZR3i4X7EwJReMKWFxaCh6Wh1/fhZXvd+Dpd/fb+rpiSmgoGi/Ye4aVNIsKi9NSQtoutwzRw1Jum4BSggIWGwlrG8cVICW0J+lSP2AhtSOmCaw8L1dYwBKJy/jLewcL/vOy0T0YwQ2PvoWBSBznnjQS//GV0+B2SWnHuV0SRlQk00I5+lj+8n4H///9w3CttUQ1ylu5pYVyMd2yz6NW6RQ7UKtGeedObO5KtrXfc8TeQLg3lNpxtlDGW1FhcYq/TwsL3kT/CqAGLD1D0bL7TJUSFLDYSFpr/gJUCTF/hZUPTarCUvibqCgx/+GdAwX/edl4a9cxHOmPYHRdEL++blaK1KslHx/Lge4hbNpznP+7GAGLthy3p8wGIIZzSAlVJm8+A+HUG3MoubuXJFVZYf91YsByKBlI7Dlmb1qiX3NdCmW87UgGXM21QcemhMTBhyIBr5urQr1DzhwpYJbv/uF9zP2PtWl/91KAAhYbCWmGH/pt3kUoipJbwDKMHhZFUVI+CO/t68b2rr6C/sxsdCev1eSmKt5u2wjmY8nFfPjX91PVpOFIv2kRU0IAKSwAUJ9UzY5r0pNsIxH0unl6MNuwxGKhKAqv8Ntz1N73lbaibzgUFm66zZASKkbqxSglBKhlzr0OroA0w4sfdmLHoX5s2Hm02KdiGQpYbETtw5Lamt+ulFDvUIwHH1Ym8Wo9LIVcCIaicW5YnTNpJADgD+/Ym3O3CrtWrPlTJhqqc+92+5f3Eumg8yc3AHBKSsgZHiK7yMXDMqIyEbBofUlqZ2pVccvUZK6Y9IVj/Jw6e0O2ppn7QsOjsLCApbk2oM5ZM/g93tl7HGf86CX8/u29BTkXI7SDD0VYEKO9XqUGC9Tf29dd3BPJAQpYbERbIml3SuiI4KvothCwiDexUFTO2VBqBvZhdknAdZ8ZDwB49t0DOVfd2EGPhYBlZCVLCVlbtHcdGcCWAz1wuyTeOdcRKSFSWHjAolVYmOEzILyWUxUWsX+Sotir3vHKmOTvXgiFJS4r3IMjKixGAcv6nUfRPRjFKx8ftv1cMpFZYfGmHFOKKIrCvZaFnsxdCChgsRFeJaRpzW/XXBJxIcnVwwIU9kYq5oAvntqE+govunrDeG378C48It2DFhSWHD0sK5Pm4jmTGnDamETn3CP94WHvM1H2KaG49T4s9UYKC9tg+ESFxfkBCwDsttF4yz6zE0YmppUXYgDi0f4wYrIClwSMqvKrHhaDlBDzGw13+sXIwwIANVxhKd3PlFgZ997+7pKreKKAxUbCRsMPbSprFtMU+QQshfSxsA9zdcALn8eFq04fDQD4w6bipYWsKCzqAERru8yVyeqgz89sQW3Qyxe84eh7I0JVQumwyq/jA6nXgrWFDwgmbF+GrrjF5JAmYNljY8k8CwomNiR6jRSiGzZrGtdYHYDH7cqaLh8IJx4f7vdvJoWFfaZLWWER70Xdg1HsLULrhXyggMVGwtoqoeRCGI0rtgz5EkttczXdAoU1g7IPMzOofXnWGADAix92Fa1ixVJKiAUsFtJm2zr7sK2rD163hHmnNkOSJIyuCwIY/rQQC1hY2XbZBixu40ovLfWVib/7MaOUkLe0UkKA2t7ADthndkKydLcQ3W47BP8KgKwelsFkx9lhD1gyeFiqy8DDom2eWGppIQpYbETbhyUoSM12DPlKSQkNmu9imqawDENKiH24T22twdTmakRiMlZuKU5PFmsBi3UPy8pkddCFJzfynzGmngUsw7uDicQS7wnm2+gp8RJMLbl0uuUeFk0Qyj6vQZ/zTbcsYGG7fDsrhdhnlrWfL4TC0plsGteSDFjUlJD+dWYKi5XiAjvQTmoWYR6WUg5YtIpWqRlvKWCxEW0fFjHPboeXQcwtR+KyaTOvVmEpZJqiX5MDliQJXzozobL8sUhpIStVQqOElJCZgFBRFN4c74rTWvjjasAyvApLTE78rVlqq2wVllxSQoMRyIL5WzusFHC+h+XM8fUAYJuUL7YhYAP+zL73raD2YEkGLFkUFjbTpy8cS/mbFZo+nhJKXyvYmlbKHhZtIE4BywlMSONhkSRJ6LKZf8Ci3fWbvRlFk29SpnoU8ibaK3hYGPOnNwMA3t/fk+axGA5YRVVthXmFJRSVMWiibfjWA73YfXQQAa8Lc09p4o+PqU8YGIuVEmLm4XILWMI5lDXXJQMWWUk1caopIZ2ApQjv00ywUuOzkgHLvmODtqSZxTYELCUUjsm2+zTEHixA9pYPg0mFRVHUIGI46OfrV3mmhNj1ZlNJth7sseV9NFxQwGIjYU2VEGBvabM2t2z2ZsQWXyb5Hjg+VDB3uJ5pbXRdEBU+N2KyMuwmL0VRLKWEKnwevpiaqRR67M3dAIBLpjbxjqpA8VNCTCnqccgsJ7uIJDcFVhQWn8fFPQlipVBI0zdJfF3HmW6TCsWMMbXweVyIyQr3heSD2IZgZKWPqwh2p4XUHiyJz0UgS5WQGDANZ1qIe/AyeFhK2XTL3tdj6oOoDngQisr4pKt0JlBTwGIjWoVF/H87FBZtMzOzgwWZvM12UH3hWMHaS2s9LADgckkpw8OGk4GIuoOsC/pMPYepLEeyzBPavK8bf0w2xbvp/LaU7zGFZThmN4nwlFB1aaWEYnEZG3Yezfo5yaWsGVBLm8VeLNq+SYBzhx+yzUpTdQBjk8GwHT4WcXaOJElcmculcWImOnv1FZZspltgeN/DPKWdoQ9LSaeEhO7OM5PtF97b313EM7IGBSw2orbmV3dsdna7ZSmh+mRqw7zCkrhh1wS8fEHa312Ynb+Ry/6k5Hj2Tw8PbzTPrpHP7Uq5MWWClzZn2GXKsoIf/PkDAMAXzxyNM8bVp3yfKSyH+sz1YhmKxPGLlz7B27uPmTpHI7Qpod5QrCR6LfzPuj34ykPrMe+Xr+LNHUcMj8vFwwKIvVhSJxMDmpSQA4cfxuIyjiaVoVHVfowfmQj+d9tQKaRuMLz89QF7FRZFUdWg5hqzHhb18eFUWPoymm7LICUUU+9Rp42pA1BaPhYKWGwi0UEwuftLUVjsSQkNRdSx7+zmbzpgERb50QX2VvSF0z0sgGro2zncAUuylLom6OXzYrLBm8dlKG1+dvMBbN7XjQqfG9+dPzXt+3UVXlQkZe+DJkzOP3txG+5bvR0//utHps7RiGgyJcRmIsVlpSQkbJY623N0EH//2w247Q/vpZXBx+Iyb3plxcMCACOSQb5YKSTuNhmsXNpJAcuxgQgUJTlNvNKH8ckGb3akV7WKaK59iDJxfDDK16AmFrD4MivP4qDK4WoeJxqQM/VhKemAhReGuHDa2DoApVXaTAGLTUTiMthGNnXHZo/CwhYQv8eF1mSPD6sBi9ftwpjkcwuVqjDqFKkqLMObEupOztKpDaYvQEawm71RafNAOIZ7nv8YALD44kl8ERaRJMl0pdDmfd149I1dAICOPCu4orJqsGY39VJICzEvA7sZ/+/G/bjkP9bixQ86+TGiETZnhSVLSijfKqGOniF883cbsd7GwXKsadzISh/cLgnjRySu0e4j+X+WtH2TRlXbH7B0JEuaG6r8/O/G+7DoeFhkWUkxvA/X+3cwEudreLU/3e9WFikhYX4WU1g+6epLScE5GQpYbEJUUMT8OptTkm+3W1Yl0FDl5+ZR01VCQu+KQpfb6nlYgOIpLFZKmhkN1Znz+L96ZQcO9YUxbkQFvjGnTfcYwFylUDQu41//+D5XDo4ORPIq42R/a6/HhRqL75NiwlID131mPJ76VjtOGlWJI/1h3PLEO9ib9GqIQYTVgEXtdqv+TfVSQqrpNreA5ddrd+KlD7vwq1d25PR8PVh6hgUT45N+MHsUllRFlCksdqaEtBVCQOaUkPaxQvnttLDgze2SdNPHoum2FNKseqjjY1xorg2gqcaPuKzgg4O9RT4zc1DAYhMscpWkVLk6007CCqxipaHKhzqrHpaY+iYdnQxYDhTKw2IgqTLT7fHBaEGHL2ph14iVtpqBKSx6u8w9Rwfw29cSasjtl5+ScrPTYqZS6KFXd+Ljzj7+N43LiqXBllpYSsjndll+nxQTtqMO+tw4e8IIPHfr+ThrfD3isoI/bT4AQH0fSxLgcZlL7zH05gmFMgQskRw2GLKs4IWtCUVoW2ef5ecbkRawJBWWPUfzn7yuVUQLkRLSdrkFEtV4gL7yPKBJYQ7X+1e8FnrpYxawyApMtTxwImyuHVP+S83HQgGLTfC2/B53ypudm27zzIkfyUNhYVK61134lvFsx1aj8bBU+Dz8Zw+nymKlpJkxMsMAxLv/+hEicRnnT27AZ6c1pX1fJJuatfNwP+5bvR0AcOfnp/EAI5/dLVdY3C7+Ow93t9BcYDcu5vvxe9z4ytljAST8QqJHzOd2mfYjMfQmNmsbPSZ+bu4Ky+b93bwa5lBf2PLEbyOYutqYDFjG1FfAJSWUiHyVEK0iWgjTbafGcAsICovOjX9A89hweVgydbkFEufMRl6Ugi9Mj5CweQVQcj4WClhsgrfl10iJTFrMd2Izq1hpqPJblvp5O3O3Sy23LVC3W22nWxGWFhrOSqFcApZRBrvMjp4hvPhhF1wScMfnp2W9aWa61rKsYOnTWxCJJYKfL5wx2pbdLfOweNwS/527izTDyQpcYRGCh/nTm+H3uPDp4QFsPdCbU1t+Rn1FupGapR6CugqL9YDl+S0dKf+2S2XRKiw+wceW7xBEbZVQIcqa9RSWQNJ0OxSNp6lExVNYjJvGAQlfWql3u9WqimWtsCxbtgxnn302qqur0djYiAULFmDbtm0Zn3PRRRdBkqS0r8svv5wfc8MNN6R9f/78+bn9RkUiJCgsIgGbTLdsoR1Z5bOusPAqITdPCXUPRm3fJcRlhe+O9D70zHg7nL1Y2DWqsaSwJE23mtTVW7sSJcczRtfi5KbqrK+TKSX0vxv3YcOuYwh63fj3L8yAJEmGgZIVxJSQ1fdJMRniKSH1fVMd8GJuUsV6dvOBlNSmVfTmCYV1TbeJz6vVTreKouD5ZDqIvfc/silgOdSXuOGz9wegmpPzNd5qb9Lcw2Jje/7O3tQ5QoAaJMpK+rXWBizDpRBm2mwx2HXqLdFKIaYcsvvUjGQvlr3HBoc1VZ8rlj75a9euxaJFi7B+/Xq89NJLiEajuPTSSzEwYPyhefrpp9HR0cG/tm7dCrfbjauvvjrluPnz56cc9+STT+b2GxUJ3oMlTWGxp6xZNN3WsRuRyZ2zWiWU2CGw1EO2SqG4rODpd/abKssF1A88oF8WWAyFhakLuaSEjg9GUtpWb0gGLGdPGGHqdVgKrKs3nNI5VZYV/PJviVTQtz97MsYmPQkNNsjxeimhUghYQjpqBwB84fTRAIA/v3eQKyJWS5oBYASb2DxgMiVk8fP6wcFe7D8+hIDXhWvPGQcA+LjDHiOjqrCoN3zWiyVf461RlVAkJtt2U9ZVWIRrHtIMQNT6Q4ZNYclQ0sxgwUx/iQYs2vtUbdDLO6C/XwIN5MzXegJ44YUXUv792GOPobGxEZs2bcIFF1yg+5wRI1IX9xUrVqCioiItYPH7/WhubrZyOo5CG7ky/DZ1uuUpoWo/n4mTS5UQkLiRdg9Gsf/4IKY0GysFD679FD9dtQ0LTm/FL685I+vPYT1YfB5XSvM8RjEVljoLAUt9hQ8uKbH7Oz4Y5Ys4U1jOaTMXsIyo9CHodWMoGsfB7hA3Hm/ccxydvSFUBzxYeO54fjyT4w/no7AIfqVSqhJiN6kKX+r75oKTR6GuwovDfWGs3XYYQH4pod5QDNG4DK/blbFKyKrC8lwyHfR3Uxpx5rg6AMDHBUoJAanG23zQelgCXjeq/R70hWM40h+2FOjroSiKUCUU5I973S543RKicQVD0Thqof4cNviQfQaHS80wo7DUlPjEZr371Glj67DzyADe29eDi6Y0FuvUTJGXh6WnJ2HU0QYlmXj44YdxzTXXoLKyMuXxNWvWoLGxEVOmTMEtt9yCo0eN+xiEw2H09vamfBUbvZ4OQPaOjmZh6YmGytSUkBnZViulj+GVQsbKyUA4ht++thMATM8sYR/iGoMdClNY9h4bHLYhiLmUNbMGXYCanjnSH8aOQwllyKzCktqLRb2xsOnOl05rTgnseA+Mvtyl2Wiyq3GpKSx6wQOQCCAun5GYgv2H5LTvXAKW2qCXD3xjqlumPixxWTE9FE5MB82f3oypzTUAEv0t7BgsxwKWxur0lNCePLvdqq351c+HncbbdZ8e5cFos6ZfUcBgbWQpocakojRc71+t2qSHWtrs/M+UHnrvedaif8sB5xtvcw5YZFnGkiVLMGfOHEyfPt3Uc9566y1s3boVN910U8rj8+fPx+9+9zusXr0aP/nJT7B27VpcdtlliMf1b/LLli1DbW0t/xo7dmyuv4ZthHQGHwL2pYR4lVC1WiUU0zRYMiIipAkAYHRd9jk3yzfsxfHkwm7W65LNZd9cE+BDEO2Yg2IGbro1MalZRG0elwgeNiZb5k9pquYlsmbQVgrF4jKe35rYjV9xWkvKsbaYbks0JTQklDVr+cIZibQQC7BzCVg8wvVglUKZypoB8yrLtq4+7DoyAJ/bhYunNmLciAoEvW6EYzJ25/k+HwjHuC8sRWFJpoTyNt3q3KTteB8qioLfvLoT1z3yFgDg/MkNaX9bo0qhgeSk5pa64gQsmRSWqhJvzx+OpvvAWGBYCtWEOQcsixYtwtatW7FixQrTz3n44YcxY8YMnHPOOSmPX3PNNbjyyisxY8YMLFiwACtXrsTbb7+NNWvW6L7O0qVL0dPTw7/27duX669hG4ZVQjY0jovGZb4rbKjyI+h1w+tObBfN9OzQzl/JVm4bisbxUFJdAcwHLNomVFokSRr2BnK5VAkBQmlzcgDiBovpIIZ2COKGXcdwpD+C+gov5kxqSDl2VJ5NuxRFQUxmCovE02BOX4hicZkHBxU6fW1mja/n71kgNw8LoBpvmY8lpNuaXwhYTFYKPb8loa5ccHIDqgNeuFwST7V+3Jmf+sveCxU+d8o08HHJlFD3YNS0l00PvUaPvHFiju/D/nAMi5e/i7uf+whxWcEXzxiNh647K+04FsBoFRbWdbU1mUKKxGRbZrFlQ+3DYrxWlL7pVmd+FvNtDZPqnQ85ffIXL16MlStX4pVXXsGYMWNMPWdgYAArVqzAjTfemPXYiRMnoqGhATt26HeL9Pv9qKmpSfkqNnoGPvHf+ZQ1swXW7UrchCRJQm1y8rCZxUosawbAK4X2G6SEntq4D4f7wjzA0br2jTBqyy8ynC36ZVnJOWDRdvy06l9haFNCK99PpIPmT2/mihcj37boLB0EJBWFEmkcJ96w9BQWSZKwIGm+BdJVTLOI3W7jssI/F+Jn1uN28V4bZnuxvMDTQapidkpLMmDpyM/HwvxMoroCAJV+D39sz7HcP0t8kyGmhIRKIavsOzaIBb96A3/d0gGvW8KPrjoVP/8/p+n+XY0Gw/YnFZbGGj9P4w1H0J1pjhCDBTOla7pNn3fnt6n1xnBgKWBRFAWLFy/GM888g5dffhltbcZtybU89dRTCIfD+NrXvpb12P379+Po0aNoaWnJeqxT4AqLRq62IyXEbpojKn1wJRdTNhvHzM2IVwlpPSw65baRmIwH1ybUla+fOwGAefnTqC2/yMQGZrwtvMLSH4nxlve5KywR9Iai+DBZ8ZGrwrL/+BCicZl7HT4/szXt2AahnDqX9vyiL0gsa86nc+5wwAIWSTIuWV5whnq9ckkJAanzhMSqLa3vjAX2ZhSWTw/3Y1tXHzwuCZ89RW0kyHwsdiksjZqABcjfeBuNy3xd0k0J5eCl+tUrO7DjUD+aavxYcXM7rmufYNivyKgLOFNYqv0ebnIdjuZx/Tx4y+5hKfk+LJ50VTHX+VnDiaVP/qJFi/D4449j+fLlqK6uRmdnJzo7OzE0pO7UFy5ciKVLl6Y99+GHH8aCBQswcuTIlMf7+/vxne98B+vXr8fu3buxevVqXHXVVZg0aRLmzZuX4681/GRTWPIx3YpdbhlW/AlRjcIyJulhOdIfSdvdPPvuARzoHsKoaj++cV4iIA3HZFMmWTM7lJMah6+0malPfo8rYwt9PXjw0B/Gpj3HoSjAhJEVuoMOMzFaSL+9vuMIugejaKjyYbZO4MOCpLispHRkNYv4N/IKjeN6h6J5zScqNKysNeh1G97cJjVW49TWRBCQa8AiKiziTdKosi9sIo3L1JVzJzWk+KSmJlNCH+WrsOhUCDHG5Wm8NWpDwE23OSgsXclOv//82SmYNb4+47FMYRlMM90mK8b8HtRY2Jjli5n1q0aYJ1SK8CohnQG9TppQboSlT/4DDzyAnp4eXHTRRWhpaeFfv//97/kxe/fuRUeHpuPjtm14/fXXddNBbrcb77//Pq688kqcfPLJuPHGGzFr1iy89tpr8PvTP6ROxahKKGBDWfMRYY4QQw1Yst/YtFVCNUEP30WIPpZYXMZ/r0mk4f6/CybynD9gLi1k1JZfhCksnx4eKPgAsVzTQYB6rY/2R3JOBwGqmtXVF8Iz7yRm4lw2vQUeHR+G1+1CffKml0unUTEl5HapAYusJNQmpzIYTZybtqRZy9WzEulnsQGZFdR5QlHeotzncXHVksECezMLODNQXzY9tSUDU1gOdA/lpQ7oNY1jTGDG2xwVFqaIBryulPRkPqZbZhCuzKBSMFiaKJRmuk2cV6XPLQTdhX//mklpF8J0G47FseiJd7B8w17bXtMI3odFCPrzGUcx3Fjqw2LmBqNnlJ0yZYrhc4PBIFatWmXlNBxJmAcF9ne6PaqjsLBhfpZSQslFSZIkjK4P4uPOPhzoHsKkxkQQ8dctHdh9dBD1FV78/exx8Lpd8HtcCMdk9IViWQcImulj0NZQCUlKnPexgQjvKlsIcilpZogDELt3JV7nnLaRmZ5i8Do+BLwuhKIy/prs1fH5mcapzoYqP44PRnGkP4wpyN5NV0RU0iRJQsDrhs/jQiQmo2cwmjGQLCZM7cimgi1sn4DR9RVZd+5GsOZxxwcjgjSeHjgyhSWbRB6OxbH1QCLlc/HU1P4VtRVetNYGcLAnhG2dfaZL4bVkUljU0uYcA5awvkm+gZfXWw9YWDonW/AJGLd8YH1YKoWU0HAqLBnLmv2sD4t957Pu06P465YObN7Xjb+fPc6219WD36d052eVmYeFMMZYYcnfw6KmhPQUFjMpoWS7dmFxZjv/tdsO41ev7MB1D2/Ad//4PgDgpvMn8mmq1RYkUDMelqDPzd3/O/NsK56N7nwCluS1PtA9xDtAnpPDTSfRiyVxY4nLCppq/BlvXlqzrxXEpnGMUihtHjJoGqfF5ZLw2WlNKcqfFVjzuGNCSkjPDGpWYenqSfyN/B6XrsdkakvSx5JHx9vMAQsrbc7tc2T0eVXN3xHLKqhRA0A9jNLl6mt4VIVlODwsJgIWrrDYmBJiAefhPvvGIRihp7D4SkhhoYDFJlTTrVZhyT96VVNC6qJltoupoii6Q+NY2/hH3tiFn67ahte2H0EoKmNaSw2ua1e7rzJp11RKyEQOGBBa9B8qrI+Fd7m12IMFEGXxCKJxBc01AYwdEczyLH3EktzLZ7SmpSBE8qkU4k3jhL+z0cTm+1dvx789s8UR3ha9IYSFQJzYrFfeyfCxeUJZFvCDPeqMHD3vDfex5NHxVp3UnJ4Ga0sGLF29YXTn4HniAYtGER2ZvE6RuGw5FTMYtpISSg5ANEoJ+d2qwlLgAZ6KoggKcfayZjtTQixgyeV6W0Wv95BfeL8XOmDKFwpYbCLMTbfaPizGY9TNwm5eI3VMt9km8UZSjJjquV1w8ihIUmIRv2x6M35wxTQ8f+v5WPmP56WkDvh0UgseFqM+LAzeor/ACksugw8ZI6tSd/HntI3IOp3ZCDFg+fxpmSvfGvIoKWUKi8el/p3rdALb4wMR/MdLn+CJDXt59VMxGTSZEsqXeqEPi9GwUsB8Tr+DByz6gawdCsuhXmOFpbbCiwnJtND7+613Ke03SAkFvG5uLj3cb67LNYOlhPSUKy1GZc1iSogZmQutsIRjMu9hlNl0a39Zs2iatnq9raKabtPLmgHrIymGG0seFsKYkMGOjRvL8pDbMptuM3+QRSOmKANeckoTPvzhfAS8row3YivDvrikmmV3ddIwKyy5pIQqfB5U+Nz8ZpqL4ZbBUkKj64I4Y2xdxmPVpl25Vwn5sqSENu45zv//w45eTB9da/ln2QlTWMykEfJBr0oooJcS8pjzsBzsTs7IqdM3AZ+SVFi2dfZBlpWMypoecVnhIzn0AhYgMQdm99FBvLevGxecPMrS62dK4TZU+9EbiuFwXwSTLIyXYZ+XSp8JhcUoJRRWX4MFToVOabJrIUn6zQsZbD0cisb5TKp82S0GLBavtxUURdH1Wor3hXBMzrnP0XBACotNhHRaHgPqDi4uKznPz9ErazbbxVRcdLUfrqDPuIyUwRYzc1VC2XPAwPArLLkOcBNVFr0yZLN8bnoLpjRV458vPTnr9c6naVemlJDYi+WtXeqcrg8PFl9h4ZOaCxywMIVlIBLnO3Zd0y0fgJhZFWUKS6uBwtLWUAmf24WBSNywq3Qmjg8mGtxJkpqm0XLamDoAwHs5TNrNVBWTS6VQRFApzPwtWbCoVZ/ZxqdiGKuEeEmzz5MxsBTVFztUlrisYN8x9b2Rz+DTbIiKoaiwiN2drU4pH24oYLEJo5y4KLflUikkywrvdJvSh8VkF1MWsLhdEu/gaQWWizZjuuU54KwelkTAsvfYYEGbFeUbsLDrPaLSxyupcmHcyAqs+r8X4ItnZu8KnU+FhpoSUv/Oel4nVqYNoCApoWffPYANO42Hl2phu/Kgt7CCb03Awz8DbKCnXhqKp4SyLN58CrGBwuJxuzC5KfG++SiHBnLMcDuy0qdbBg8kFBYA2Lyvx7L/QN1gpH8+chmAOCiUzudaJRSLy/zGWuX3WJ44/uHBXjz8+i7LKXiza5fX7eI3ezt6sXT2hlLSMHYMnDRCfD+L73tJknKeUj7cUMBiE0bDD/0eF28vnUulUPdQFPHkrmWkTkooWxdTbdM4q3APi4ndRKYFUKSpxo9KnxtxWcHePNqKZ4MZ9XIx3QJqafPZE+pz9q9YZVQePTDEwYcMbUqoPxzDVkFV+ehgr61Gu/f3d2PJ7zfjKw+tx11/+dCU2Vyt2CnsciRJEq8UYoMU9Yy+ZhdvlhIyUlgAoeNtDg3k2M2rIUPp/6mtNfC4JBzpD+OgyanqDNVzln6TzuV9yAJPn9tlKlWi52ERm8hV+N08YDHrYfnBnz/Aj1Z+iGt/s97SzZ+VeGdqycCotrH77h6NylzIgIXZFlxS6qYGEIN0Z5c2U8BiE0bDDyVJ4m+GXBQWtmDUVXh1b0TZupiGhQZZuVBlsqw5HIvzBT7bhz4xBDGx89xxqIABS54KC1NVLppSoKSyDvm054/qVINpA5Z39hxHXFbQUhuAz+1CXziWU7rCiNe2H+H//8gbu/CFX72JHVm8SqqHpfCWOtaLpSMZsGhN8oC66cimsLCUUHOGRnZ8plAOCsuhDCXNjIDXjanJn/Hevm5Lr58phZuPwlLhN5fa0xt+yFLPHpeUMl7CrMKyNzm9evO+bnzhv9/A9i5zgWKfSYUFEFo92JAS0k7bzmdCdjbCQjd27QaMv+cdXtpMAYtNZKo6MHLDm4FXCGly2Ga7mGqbxlmlymfOwyIqMGZ2Kay0eXeObcXNkG/AsmTuZPzhW+34P2eNtfO0MpJPe37mYRF3T9qy5rd3J9JB7SeN5OmKDyz4WHpDUcQyKA9vfpoIWK44rRUjKn34sKMXV/zn6/j923sNlRyzjePsgCksmVJCfLZKht9zKBLH8aSCZ0phyaG0OVMPFhHuY7EYsGTqO8IM/lZuoLylvsm/o94soQGhLFqSJHWWkImAJS4r3APSXBPA/uND+OIDb+J1IYg2wkzTS0a1BdU5G2z9Y3+D4VBY9OZ1lUq3WwpYbMKocVzisdybx+n1YGGvyd5kmXoUsF230VC5bJhtlMQ+8JU+tymvTHNyJk+uI+zNkG/AEvC6cdaEETl5f3Iln/b8eimhOo3XaUPSvzK7bQSmJctuzfpYNu/rxtk//htuSzYY1BKKxrFxd6IC6dZLJuP5W8/HnEkjMRSN47t/3IK1nxzWfZ7qYSl8wMJ6sRzgCouOh8WbffFm6kqFz83n3ejB1I/dRwdSPB5mMB2wcB9Lt6XXz9SGoDXZp0mrAGSCN3wzcdMHRA+Lep3ZNapMqi/ss9sXjmVVHI/2hxGXFbgkYOU/nYezJ9SjLxTDDY++hT+/dzDjc800jWOw68XSSPmw50ji+p45LtG9uaABi04PFobfZGVcsaGAxSaMWvMDQsCSQ/M4dkNv0OvDYEIu1WsaZwWzZc1m/SsMcRpyIZBlheeYc+nDUkxy7XabLSUUisb5Te2ctpGYlhwmaLZS6GertiEck/GX9w7q5u/f2Xsc4ZiMxmo/ThpViaaaAP7fN2Zj7imJlNoWg14hoWEqawbUSiF1jk6mTrfGn1em0Bg1jWM0VPnRUOWHolhTsoDMTeNETk8GLFsO9HC/mxkyVQlNaUoGWkcGTCvDVtryA0LLB+H1eYVQ8pxYMKgo2RWNzuTgxVHViWv++E2zseD0VsRkBT9e+WFGrxavEjIRbFlp9ZANFhCelRw3MRxVQvrNEkujPT8FLDaRSWHJx8NydCAZsOiUNZoKWHhKKDeVoMpkp1tuWjOxQwFSZ/UUgr5QDGx9ylVhKRa5drvNlBLqGYri/f09iMRkNFT5MWFkBVdYPjKhsGzacwyv7zjCf84rHx9KO2bdp4nKoHNPGslv4i6XxIf0GaUuebOx4VBYNPOw9D6vZvqwHEwqNEyJyAQriX9jR/bUhMgh4QaciZNGVaEy2TMom19IJJuHZUSlD7ICbO8y95pW2vIDgsIipIS0nXL9Hjf/G2UzuXb1qukg9tx7vjQTHpeEQ31hrqrp0Weiyy2DXa/ePAMWRVF407izkuM6mEpUCPTa8jPMVsYVGwpYbIL9of06i65ertYsrIGYXqWAVu7XI2+FxaTp1mwPFsZIniMvjMLCrknQ63Z0IyQ9cp2Wm61KiJUaz0527T2lVZ0onK21+32rE1O82c3oha2dacewG/K5kxpSHq/KYlIcGqY+LICqsDD0giQzBsROQWHJxnmTE9fjNRNeChG229ab1CzidkmYMSbR/G/zvuMZjxXJlBKSJImrLGYNw2o6x9waoDdLaECTEgJgegAiU1gaa9S/ScDrxinJwDxTyqzfwoaLXS8zZc27jgzg4p+twW9f25n2vcP9YQxG4nBJCZVMkhKeRKveNbNkukeR6fYEQpbVeT16jai46TaHNwNvGpdrSohVCRW4rNmKaQ0QqmEKpLDk618pJrm2548yNU14D7J0mKIAq5OqCOvaWxPw8vlImXws7+49jlc/OQy3S8LPrj4NALBm2+GUALw/HMN7yZTPuSelTrWuytLLh3kYhsfDkt6GXosphYUHLNkVlvOSAdzmfd2WSmHNeliA1H4sZlAUJatvYyqvcDJnGB7MMFBSD/0qofSKMaN5WFqYItVUk3q9WMps895uw+ey9Stbl25A8PWZ+Fv+9rWd2HlkAI++sTvte3uTM4Ra64II+txc/SuUjyWj6ZZNKM/SLLHYUMBiA6kdBPUUlvzLmvUUFjNNlfKuEmIpoSyGQfbhrbHoYTmWQ/muGUo5YMmlpBQA7zIqBqeiOVv1r6hde7nxNoO/4v7V2wEAXzhjNC6b3ozRdUEMReN4dbtqon171zHEZQXjRlTwUQSMbDn/IYveh3yoN5ESMmNA7BAGH2Zj7IgKtDVUIi4rWP+puYZ6oWicbxIaa7IHLKdbrBQajMTBPnaGAYswWsDsawLmFRYWoEZiMk+DMJWmSiiNNts8jqlezTWpf5MzxtUBAN7NqLCYL2uuMVnWPBiJ4c+bE2bfA91D2H881cC8OxmwsJRprp97s4Sixh4WSgmdQIhGJb3olb1BcmnKw1Im2mF8gLkBiHpGTCuIcr5dpjVArdaICeZYOynlgKUhx3RZRKfTLZB6DWoCHi71A8C0lkQqwUhheW9fN17ZllBXFv/dJEiShPnTmwEAq4S0EEsHzZk0Mu01sqUV2Q5bb66P3Yyo1AYsuRkQO/gcIXMTvJnKYjYtxG5afo/L1K6fKSzbuvpMpZ5ZMOR2SYbK1hSLJdlWBh8CqYoa28xxhcWvo7Bk87Akr1mjJmBhCsvWAz2G41EyGZC1mFWdn9vSmVJdyVoKMJh/ZVxygGWuZnuz8G7sOvcCH5U1nziwyNXjknRbaOda1qwoCldY9PLYVlJCOZc1Jz+cMVnJ+Ga26mHxe9z82EL4WLqHEq9Zm2OX22KSa3v+aCx9lhCQGrCcPWFEyqyUbJVCTF256vRWTGhI7ARZwPK3j7r4++vNpHLQflJD2mtkTQlZNGvmg5WAJXNKiM0Ryq6wAMD5SR/L6yaNt6z8vLUuaKrLckttAKOq/YjLCj44mD0txBTRqmS/Ez1ObqqCJCVUXjN+KrWHirm/o7gmsaBV38NibgAiSwlpFZa2hkrUBr0Ix2TDjsNWFBZe1pwlYFnx1l4A6udPHIkBAHu4wpIIWHI125slZMrDQimhsidTfXvi8dxSQv3hGA8S9BQWMwMQIzpGTCuI8m4mk1mfhQ88o5A+llJWWHIdgGg0hkEcTaCdOn1qMmDZcag/bbHasr8Hqz8+BJcELP67SfzxM8fVo6EqMc13/c6jOD4Q4QpN+8R0haU6m8JShD4sjIzyuEHA0h+O8ZuVWYXlMyeNhNslYdeRAezL0tskLiv471cSJuevnG2uaaEkSbyBnJl+LH0m+o5U+DwYPyJxMzWTFlIDT3NrgMsl8bWRPZdVI1bqKSxZBiB2cg9LasAiSRJXWd41MCWbnTQPqNcsU2+qHYf6sHHPcbhdEm6bPwWAGoQymMIyfphSQpkUFurDcgIRzqJi6LnhzXA0qTxU+Ny6i4CZAYjcdJujwuJySXy3kylna7UPC6CmPgrRi6WkA5bkwqXn71m/8yg+PaxfZhqVs6eEtAFLS20AdRVexGQlrXz1l3/7BABw5WmtfJQCkEgjXHpqEwDghQ86sS5ZfTSlqVrXIFqZwcOiKAqfHzMcAUtQ8PQY/cxsizdr618d8JhOgdYEvPymmU1lWfn+Qew8MoC6Ci++9pnxpl4fAE4fm0jvvWfQ70bEbAqEdeo1U/o+kIMXSdsFXM8HY8bDEorGeWpcq7AA2Y23/RbWLzOm29+/vQ8A8HdTGvH5Ga2QJGDn4YGUYIR5WMYzhSWPSe1mUBUW6nR7QpNdYcktJZTJcAuIAxCNb/iRPIcfAuZKm/szDFIzgvViMVJYNuw8ivN+8jJe+rDL9Gsyeks4YGEqgLY9/9YDPbj2N+ux6Il3dJ9nlBJiC37Q68b00bUp35MkSdd4u/qjLqz++FDCu3Lx5LSfNf/URFroxQ+6uC+j/aR0dQUQcv46759wTOb9coajrFmSpBSVJdMsIaPW/KxCKFNLfj14WiiDj0WWFfzXywl15cY5baYDIkD1sZgx3po1yU+xYLy12ocFSJ/YrDaOU1/DjIflUK/q+dHrPHx60nhrpD5ZUYhrsqyHkZiMP75zAABwzdljUVvh5b6xjUkfS/dghAdg40akpoQKprCw+5ROmwfysJxA8IY8OosfoL5BrHa6ZY2O9HYMgMXGcTkqLICwQ86UErJQFsjI1otl1Qdd2H98CH99P3NbbT1KWWHxul38pirutv7y3kEoitq0TIteHxZAvQazxtfrpga1LfqHInF8/88fAABuOq+ND4EU+czEkagOeHCkP4xn3t0PIL2cmVGdbMYViclpqoWYJh0OhQVIrRTKNK3ZqGKik1UI1ZnzrzBEH4tRc7AXPujE9kP9qA54cP2cCZZef+boOgCJAYDHsqiW/SY9Z7xSyMQQQbXTrfk1gBmtWUpIrRISFBYTfVi6+pL+FYPOw6yKaueRgbSeQ+FYnL8vzZluVQ+LXiHC3z7qwrGBCBqr/bhoyigAqrLJ0kLMv9JU4+fXq/ApoUxVQmrFlpOhgMUGMrXlB3L3sOzRSIZaaoOJhdfMLKF8FJZqE62oc0kJjeSTifU/oJ29iRtDLtOEmTxcV4KmW0CoFEo2DlQUBc9t7QBgvAuKJVNCXk1K6KzxIyBJwBWnteg+T2u8/c+Xt2P/8SG01gbwT5ekqytA4qY+95REWigUleGSgNk6/hUg1YSp7ZjMduU+t0vXsF4IUhWWDKZbI4Wl23zTOJHTxtSh2u9Bz1AUWw+kp21kWeEm56/PaTPdIoBRW+HFxKQx+r393RmPNTudeGoymN3W2Ze1A6sdCoteH5YaE149VtLcZDDGoL7Sh7bktdGqLOK6Zmr4YfKaxWVFN82/IpkOuvqsMfw9zQIWZrxlQw/Hj6jkz8u1YaRZTHW6JdNt6bKtsw9//5v1+PvfrM94XKa2/InHzY2r18Le1Kw6Q4s4GMxoMcm3SggwmRLKyXSb9LAYKCxsXkumltpGsN1Yqc0RYmgXrw8O9mLfscR1SKRR0v/eEYOU0OUzW/DBD+fhK2eP0/1ZPGDp6MW2zj489GqiK+cPrjw1xfyoZV4yLQQAM0bXGqpZHreLfza076GhLJ+dQiB2u82YzzfYYKg9WKylhDxuF0+b6flY/vZRFz7u7EOV34NvWFRXGCwtZDS3idFnMoU7bkQFAl4XwjGZm0SNGLRougXSPSzcdOtLTwllVFiY4TZDEHm6wZDIl5MNFZtrAqYGnVb43GCHaTdx+48P4rVkfyJxyvs5ydb7H3X2omcoypvGiZtRprAcH4wWROnIOPzQm1lVdAoUsGSg0u/Gm58exVu7jmVUR0JZggLtLsIs7E3NcpxaaoUupkYGsHwbxwGqAS5TwNKbl4dFP2Bhu6bO3pDlD3App4SAdHn4r1s6Ur6vp7IYpYSAzDeRk0ZVwed2oT8cwz88sQkxWcHcU5pwqRCQ6HHhyaN4oKFtx69FlNFFrFaW2MEIQXXLlBIyUlg6LLTl18LSQq9qJlcrioL7X06oKwvbx6OuIr0q0Ayjk1VL2VJCapVQ5s+H2yW26M+cFhrkFT4WFBZNt1tuuhVTQsHss3t4wJKhKzBvICcYb2NxGb9KVmR947wJps5ZkiSuxGjP6amN+6EoifQoq/4BEr1h2hoqoSjAO3uOq03jhM1oXdDLDfN6qvPHnb1ZK8wyoaaEdPqw8IGfFLCULKPrghhZ6UNMVjJ+WMNZTLf+HFNC2k6IWnweF19wjXYfkeRAvFyrhIDsCktKm++cPCzpH85YXOaLkKKowYtZSj1gERUWRVHwvImAhaWEfBYHXXrdLpzcnPCpfHp4AEGvGz+4clrW5wV9bvyfs8bC65bw+Zn66SaGUWnzcM4RYtRnSQllm6tiZfChlvMmJzwN7+w9npIe+9tHh7D1QC8qfG7cdP5Ey6/LYNcx67BSC43SmPE2a8CSw9Rtdc5a4lrzPiw6ptvMCktiDdGWNIswheW9/d1cofzL+wex++gg6iu8+Ops8xVZRvOEnt2cMNvqlaOfPSExkXnDrmNq0zhhM+pySYbN4470h3Hlf72Bax7KrPZnQk0J6SksNEuo5JEkdajY+xlywkxh0XNfA2KVkPmAZSAc4zfycQYeFiD7hznfsmYgu4dlIBLnlR65lDXrBSyH+8MQs1zattaZiMsKX5BLPWA53B/Ghx292H10EH6PC8xPqJeuYCmhXLwgzHgLALfOnZzWXt+I719xKjbfeSlOba3NeJzR1G+2ozYK9gsB87C4XZKuGpWpYkJRlLwUlgkjKzCmPpiYeL3tEJ7auA/XPLQO3/zdRgDA1z4zPq1XjBVYKmUwy1rDPss1JhRR1vF2W5YhiIM6/pNspHtY0o27LK0bicmGa2iniZTQ1OYa+DwudA9GsfvoIOKygv9MVmTddP7EjOlPLdU6pc09g1HuO7xoSmPac85pS6QD39p1FHuO6W9GjZrHvbevG5GYjAPdQ6ZmGOnBvZYZy5rJw1LSzOQzOoxzwuFsVUI5lDWzN359hTfjTTfbxOZ8G8cB2auE+oU231a8CCwl1BuKpaV8OjSKihXjrfiBLtWARV24Inh+S6IF/kVTRvGg2GpKKBvM+3ByUxVuPK/N9PPcLsnUQm9U2jycXW4ZrErIqCqJyeMRHa9Q71CMB1lWPSxAYhPE0kKLl7+L7/zhfazfeQySBPzdlFFYdNGkLK+QGXajH8ymsISNJzVrOcWEwhKNy3ytsfK3ZMeGoolKnWhSERbfU1U+D/eMGBlvD5lICfk8LswYrU61/uuWDuw8PIDaoBcL282rK4AYsKjX+aNkQDe6Lqi77sxOGm/f39/DFRTtZpRt4rQKy9YDarDIlGerZFRYSmRa8/AljkuUmck3+JYD3YbHhLMpLMno1UpZ895jbM6EfjqIwXYfRvOEojYoLNlSQqKBz0wbcUZt0Au3S0JcVnBsIIJmYXekTQHtt2C8Zdei0ufOK1ArJuLC9VwyHfS5GS3YsOsYhqJx3Z2mGrBYSwkBwNWzxiIak3Hpqc0FuWZGzePY7zFcJc2AqrAYBdfixiMSl1MW+I5k5Vp9hTfnNNYlU5vw5FuJSpK2hkp86czR+MKZY7j/JB9Y/5LBLPOEckkJ7T02iIFwTDdAFX+epbJmnhKK85LmxGuo19blklAd8KJnKIqeoWjarCBFUbjC0pxF9Tp9bB027TmOTXuOY8PORMXOjee1WVKGASElJAYsybYApwhqpciY+iBaagN8M6a3GTUqbRbHLXT0hDCpsRpWUYcflu4sIQpYsjAz2T1yx6F+ww9rtiohtrCFTAwlY+zWzJkwImtKKHkT8xewrJk3XbIgqQKJhWhEpQ+H+xKzSsTFJl1hMZ8SKnX/CqCmhLZ39SEmK/B5XLh4aiOWPfcxgKi+hyW5O80l4PB5XLhhjnllxSqqhyX1fcpudMPpYZk4qhJul5RiihQRWwBEYpqApZvdGHMPLi45pREPXTcLI6t8OHNcvaUgPxvsRp8tYDHbhwVItB8YVe3H4b4wPunqwxnj6tOOYcGGxyVZ2hyJplu2IfJ5XLq9hHqGorrN43pDMX4zzuRhAVQfy1Mb9yMckxP9bs6dYPp8GarpVj0f1hZgWot+MCFJEs6eMAJ/fi/RV0rv/WccsKgKi3ZtNAtvzZ9Dd2enUJrbz2GksTqAltoAZAW6vRMAMTeYxcNi4c3A50wYVAgxzHpYvJ7cF0UWpA1EjBQW6z1YGCMr9dvzs+ZcjckP8AELKaFSL2kG1N87ljTyXDB5FKoD3owGbjvSf4VCHYCYet5DRVBYWmqDWP3tC/Ho18/W/b5Y7addwK0OPdRDkiRcemozZo0fYWuwAggpIYPPKqPX4md2apaOt7n0YAFSPSzsNfQ2PqxSSG+dY+mg2qA3qxeKVQqxNfvrc9py2tjomchZSoi1CdBDHI2htxnVa89/fCCS0tqhK8eAhbfmpz4s5c0MnhbSD1i4wmI0S8jjTjnODGrTuMwpoWwDENXW/LnfELKNU+/Pocstg+0otO35Wfvzs5P9C6x4WMpBYRlR6YN4L7t8ZqLE2JyHxd6boB3wtGJaWbP1+TN2MKGh0rAxmyRJhmWeTGGx2uV2uDCrsJjtw8LIVtqci+EWEPqwROKC4Tb9vZBpAKI69NDYv8IYXRfk6mU+/W6qNB6WWFzGJ8lZXEYpIUD1sQD66f5RycZ3rGEkkKquAEBHjh6WTJ1us3V3dgoUsJiAz+gwaMaktubP3ulWr+GXHnz0eIM5hcXIw2JHlZAVD4tVuMLSr1VYEh/KWeMT8nNnbwgxg74YWljAUqpdboFEpQ8zh3rdEi5JdpXNpLDkkxIqNKrCkvo+HcrSEqBY+Awk8oM5No0bLlSFxThgicRkfvMy+5llHW8/NqgU4m35LfRgAYTW/NE473JbqRP0ZGrPb6akmSFJEs5pS6wp+fS7YefD1r6dRwYQicmo9LkxNkOF3aTGKu6h0lNYuHdN2MAx/wozHltt8cDgxSG6CktpmG6dt7I5EK6wGJQ2Z5vWzAIZWQF3wWciHIvzhXHciMwKS7aJzWrjuNx33WwWjFFvh1y63DJYe/4jmkZJ7EN52thaeN0JY26nyZ1FOSgsgCoPnz95FF8gC1UlVGiqDCrNck0lFBqj6bVMYWl1vMKSvSs1YN53JqaE9DZddqSE9HqwMGozKMm8aZyJgAUAbr98Gu7+wnQsmXuypXMV0aaEmH9laksNXBm65UqShJsvmIhTWmpw4cmj0r6v52HZmnzts5Jqc64ellAmD4uXBeiUEip5ZiZ7sew+Oqg7tyfbtGYxP2+m2+2+Y0NQlESVC4u4jcjmYeGzhPIafpg4fyPTba8FA5+WkTrt+eOywheh0XUVvHrCbFqoXAKWiaMSwepVp7fyx8x5WByYEjJIKxajSsgMRgpLrm35hwumTkTjCv/sa2GqQNDrNt2zZ1JjFVxSom283nC+XNrys3MAUquE9AobajKsc2ytMBoSq6W1Loivzh6fn+qseT+rFULZq3e+deFJeP7W8/lmTYQFLP3hGL8eTGGZe0qitwvz91khLit8s5zJdEsKSxlQV+HjMx/e1ylvzqaweN0Sl/OM5pOIiCXN2Ux5mT7IgFAlZEdKKKI/nbSfl0haDxAaKtM9LEf7w4jJCtwuCaOq/Rhdn7g5mDXesqCy1AOW719xKh66bhauPE0IWDIsLI5OCQX0G8cNFaFKyAy8aiKufl7FpnGtDg1YxOtolBbqy2GDEfC6+fBAPR8LV0esKiy+xHVOVAkZqzRcYdGpEuKDD014WOyimqeEkgpLB6sQytxAMRtVfg+3EBzpi6A/HMOuI4n7AUsLHx+MWu6aLppp9e4FYlmzWdtCMXDeyuZQWFrofR0fSzaFRZIk/r1M8zAYu4+YK2kGVNNt9pRQPmXN6swivUUwLw8L73arKizsptBY7YfbJWFMXeI6nGgKS3NtAJee2pwStPIW2hn7sDjvY12dJSXktIBFz4R4fFAtJ2+qHb6boxV8HhefR2OUFsolYAHUUQR6CkuuM6FS+rCEMygsAeMqoa4+8x4Wu1AVlsT5fNSRCOLMKCyZkCRJTQv1h/FRRy8UJaEeTWyo5IqUVR+L2LQ00zgKwHiGlhNw3srmUE5LdrzVa9GvNuQxXnRZDvi/1+zI+rP2Jts2Z2rJz6jJsPMA7EkJBbwudTqpjo+FzxHKw8MiKixMdmd9WbjC0m2uFwsPWHI01DkZP29CmL6oODolZFQl5PCUUFhYvNkMoYYqn263UKeQrVKI3WSrLLYhYCbw44PpgxUHcqz2SvWwZDDdZqgS6uqx5mGxA7HT7aG+EI70hyFJapO9fBDnCX2QrEydProGkiTxcRBm/XwMprB43ZLuROpMpfxOggIWk6gzhdIVlmwpIQC44/PTIEnA0+8cwJufpo+WF9md7MFiNPRQhC0QQxH9CiR2br48dt3idFK90uZcd2yAWiV0ZCDCz187q2VMvTUPS3eZKCx6BLjCUlqm20oHteY3A6+aEK6z+r50ZjqIobbn1w9Y2AbDzBwhkUxjQAYzpHMyERRa8/Oy5gymW+3PjssKr6gZzoClRhh+yNSVtpGVtkwdF3uxMMPttOSsLvY7Giks/71mB37+4ra0x9UeLAYDeoV7l5N9LM5b2RzK9NG1kKTEonWoL/XNkm1aMwCcMa4eX509DgBw+7NbMzbo4T1YsjSNA4AKb+IDEpMVXSnPjpQQoOZs9SqF1E63OTSOS6aEIjGZL6SdmhsDG8R3wER7/nAsjk8PJ/ohjK139o0lF1SFpbTKmquF4YdiYO3YsmY2TyguBizMcOvMCiGG2p4/c0rIamfqugwKCzfdWnxNPdNtVSaFRaMkH+0PIy4rcEnIWqBgJ0wxHIzEeUPRUzI0jLOCWCnEerBMT742e+/pVQr1DEVx7wvb8J8v78AxTSNOtcut/tqQqfeQk3DeyuZQqvweTBpVBQDYolFZsrXmZ3xn3lSMqvZj5+EB/HrtTt1jYnGZt6Ef35BdYRFz/0M6EnDUBtMtIFQK6QUseXhYKnwevitjlUJahYWlhA52DyEuZzaEbT3Qg0hMRkOVj5sEywkjhUWWFd4V18kpIVlJrZTL1ftQaFg1lugVYgpfqw0zfwpJtpRQrilc5pc7rlMpyfuwWAw8xZQQN93qBD1GCgvrwdJQ5c9pSnmuiMHe27sTM4mmZWgYZwUWsBzsHsL2roR6c2rSQ8nS5HqVQjsO9fP/1waV2RSWxPec357f0l942bJlOPvss1FdXY3GxkYsWLAA27aly08ijz32GCRJSvkKBNKHV915551oaWlBMBjE3LlzsX37duu/TYFhaSFtAzk1JZT5w1ob9OKOz08DAPzXKzu4+1ukoyeEaFyBz+0yVaaXmLvBTHapC1QsLoPd3/PxsACZu93257hjY/DS5mQvFqawsA9nU7UfHpeEaFxJU7e0vLXrOADgrAK0PXcCRgpLVFYXGW+ef+tCEPS6VR+U8B5yrIdFR2Fh/rV8jZWFJlvzuP4cFdH6ymTQoBuw5KawsMZxsgJ0J2+yepVGYgpGFjYtZoce2o3P4+KfxY27E2uOXe8LFrC8seMIYrKC+govHwWRSWHZcUit3tIGdmpzU+O1gQfpDu7FYmllW7t2LRYtWoT169fjpZdeQjQaxaWXXoqBgfQbr0hNTQ06Ojr41549e1K+f++99+L+++/Hgw8+iA0bNqCyshLz5s1DKJRbg5xCwYy32gZyZhUWALhiZgvOn9yASEzGHc9uTfOdsHTQ2BFBXXOUHmyx1y5Q4mKbb5qgKlNKKMRy4rl5RkYmS5tZpdBBjfTucbv4gpSttHljcrdz1oT0AW3lgJHCEhMaEnpdzgtYUnxQwntIrRJy1jlr+7DEZYUrq6zztVNhCovR7C91g2EtSMycEsrPdAuon3/9PiyJxxQlddPEerA0Vg9/mq5aCKKAzC35rcBMtywoObW1lm++2NBNPdOtqLBoAxbelj+jwmLsj3MKllaJF154ATfccANOPfVUnHbaaXjsscewd+9ebNq0KePzJElCc3Mz/2pqauLfUxQFv/zlL3H77bfjqquuwsyZM/G73/0OBw8exLPPPpvTL1UoROOtGGiETCosQOJa/Oiq6fB5XHh9xxE+uZPBDLfZZgiJsB2VNiUUjannmL/Cop8SisVlvkvOpdMtoH5Aj/ZHIAtN48SJuGaMt7KsYOOexG6HzSAqNwwVlpTg1JnKUpXO1G+usDgtJaTpKLz9UB8GInFU+tyY3OhshaXSYD1g5NqZui7DGJBcO9163WoZNqsU1Ot06/e4+YZQ9LGoa8Xwl5mLpuW6Cq/pxnXZYAoL49TRaiDUnMF0KwYs2o7AZhQWXwk0j8vrLtbTk9hxjBiR+ebQ39+P8ePHY+zYsbjqqqvwwQcf8O/t2rULnZ2dmDt3Ln+strYWs2fPxrp163RfLxwOo7e3N+VrOJjWUgOPS8LRgQj2Hx/Cx529eOyNXdxXYUZhARKD1/7x7yYBAH66alvKzYaVNI83UdLMYCY77Y4qnGx6JUngi0KuGLVWz6XNt5YG3u02jKMDEUTjCRNdo/DBHV2X3Xi7/VA/eoaiCHrdGSemljJGCgtT0yQJppW54UbbPC4uK1zBcFxKSKOwbN7bDSCxaXHq9WUEsyksGfqdZIKVNXfrljUblyRng/3tmVHUyM/ENjZv7FCrLHlb/iIoLGLAN62lxrYU9ChNB9xTW9VmdExpPtwfTutkvOOwscKiDug9gTwsIrIsY8mSJZgzZw6mT59ueNyUKVPwyCOP4E9/+hMef/xxyLKMc889F/v37wcAdHZ2AkCK6sL+zb6nZdmyZaitreVfY8eOzfXXsETA6+Z19nP/Yy3m//I1/OAvHwJIVEFYWQBuOn8iRlb6sP/4EP60WVVZdh8xX9LMEEubRcQKoXw/TCzfrQ1YmDzr97hyVnFUD0uE7xxGVftT0liqwmLci4WZ384cX+fIShk7MBoDL1YIOdW7o00JieZb55U1p17nzfu6AQCnj3V+qrHSYD1g5Oo5Y2XNA5F42k0tn6nbzMfCTONG57WwfTwA4N+f+wiHkoEKH3xYhMot0bRsVzoISFdYpgubr5GVPnjdEhQFOCQ08AtF4ynqs9ZnxH2WmTwsBmuLk8h5VV+0aBG2bt2KFStWZDyuvb0dCxcuxOmnn44LL7wQTz/9NEaNGoVf//rXuf5oLF26FD09Pfxr3759Ob+WVc5JjgcPx2QEvW6cP7kB/3LpyXh28RxLN8mgz42bzp8IIFE7z1Qa5mEx0zSOwUqbtR4WNjvCb8PNu8pgnpDagyX3nieqhyUsNI1LrcQwkxLi/pXx5ZkOAowVFt6DxcG7f+aDYu8hdkOVpPyr2OxGu9tUA5a6Ip2ReVh6bSBLHxarVUI1AS83TncPpaosAxkqfLKhVdeMgp5vzGnDzDG16A3FcMefEv4/q4MP7UQMrOwMWAJeN//bVPrcKZtXl0sSerGoa+Gnh/sh2iG7tR4WUwqL8yc256ThL168GCtXrsSrr76KMWPGWHqu1+vFGWecgR07Eh1fm5ubAQBdXV1oaWnhx3V1deH000/XfQ2/3w+/vzitsZfMPRnTW2vRNqoSM0bX5rWT/9pnxuHBtZ9i5+EBPL+1A5fPaMGeY9YVlqDBhFa22ObrXwGETqUGKaFcSpoZ4gBEZiZr0SxAZuYJvb27vP0rgPEuiAcsDrvxi2jb8/M5Ql6341QhMSU0EI7hk2R5aSkELFxhieqnhFhKzmr6xuWSUBv04vhgFN2D0RSja66mWyA9YDFSqj1uF37ypZm44j9fx6oPuvD81k7Lgw/tRNyk2V05NqrKj75QDNNa06c/t9QGsP/4UEqlkOhfATKYbk15WMpEYVEUBYsXL8YzzzyDl19+GW1tbZZ/YDwex5YtW3hw0tbWhubmZqxevZof09vbiw0bNqC9vd3y6xea2qAXX5o1BmeOq7elGdvX50wAAPzXyzvQ1RtGKCrDJYFPKDZDJW8UZZwSyhejlBBbMEZU5t60ieWmj/SHcbBbv0xxbLJ53P7uoZSyRsaB7iEc6B6C2yXhjHF1OZ+L02G7oJDWwxJzbtM4hraXj1NLmoHUIZNbDvRATs5zGe7y2VzgHhYDhaUvR9MtoFYKaY23uZpuATUlxMiUWj+lpQb/cNFJAIA7nt3Ke8IM5+BDBtuked2S7UbshmRaSPSvMHilkE7AwsrxDcuaTyQPy6JFi/D4449j+fLlqK6uRmdnJzo7OzE0pO56Fy5ciKVLl/J/33XXXXjxxRexc+dOvPPOO/ja176GPXv24KabbgKQqJpZsmQJfvzjH+PPf/4ztmzZgoULF6K1tRULFiyw57d0MDecOwGVPjc+7uzDo2/sApBQE6yoIkGDlFDEhjlCjEqDlNDOwwlFaGIeTdpSPSysOVfqjaG5NgCXlPgwHRlIH77G0kGnttZYNhOWEgGDXgkxuQRSQpqgl+3KnTb4EEhVWEopHQSoN3zDPiwsjZtDZ2rmYxFLm+OywnfwuTQADGp2/dkC2EUXT8LkxiocTZp0fR5XUcZwMMXwpFFVtqyxImz23Gcmjkz7XnNNatkzoAYs05MVRekBS3aFRe3DUiYBywMPPICenh5cdNFFaGlp4V+///3v+TF79+5FR0cH//fx48fxzW9+E6eccgo+97nPobe3F2+++SamTZvGj7ntttvwj//4j7j55ptx9tlno7+/Hy+88EJag7lypK7Ch+vaJwAAfvt6ImAZP8LazV813RYuJVRtkBLaeSTxQZmY7AKcC8zDcnwwwj0qWg+LV2ikp+djefsE8K8AxgpLKaSEtAMQna2wqPl8ViF0eokodxUGKWIgEVyw665XPpwNtbRZDVjEn5NvSijodWetwvJ73PjJl2eCZRGbawJFSSm2JFXwM8bZb8T+3udOwV8Wn4d5pzalfU+vFwsLWGaNT5yLtqxZbc1v/Pfhrfkd3IfFUjisN1xPy5o1a1L+/Ytf/AK/+MUvMj5HkiTcdddduOuuu6ycTtlw0/lteOzNXfwmZKWkGTBuxR2xcRge2x1rG8exuT0TR+WusNRXeCFJiaZQH3cmvAJ681rG1FfgYE8IB44P4UzNIrGR+1ecX8WRD0YKSymkhLQelhDvweK8gEXsScFmxZSKwpKp021KG4IcUkL1Oikh9nNcOZqnxb+/WXX0zHH1+Pq5bXjkjV0YO6I4oxK+cMZoVPk9OG9Sg+2vHfC6ed8vLXxic1JhicVl3r9r1vh6/Oa1XYYKS6a/jxqkO9fDUr7aeQnRUOXHteeMw6Nv7AaQS8CSrApIaxxnf0pI7FKqKAp2JVNCJ+WhsHjcLtRX+HBsIMIXVD0T3ej6ILA7XWHpGYxiW9IUeVYZG24BY4WFpYTy7bdTSNgNklWWDUacq7Cw3ea+Y4Po7A3BJQEzRuvfQJxGJoWFfb58bpepRpda1G636QFLpc+Tk9Ih7vqtqD63zZ+CltoAzptsf8BghoDXjStOax32n9usCVj2HBtENK4g6HVjarN+SogFIf4MnzWWEiobDwtROG6+YCJfJMfZlRJigw9t2HWzfHd/SlvsMAYicbhdEsaZmCydiZGCaVeS9MsUWWnzge7UXiyb9h6DogBtDZVpPQzKDVFhERXPqI1+pUJRyRWWxGLKq4QcqLCwxZsFwic3VZeMNyrT8ENeIZRDOghQPSw9Qlkze81c/45iwGrFAxPwuvHNCybaWlJcCjCFpas3hLis8HTQSY2V/O8zGImnNJYzp7CUmYeFKBwttUF8/8ppmH9qMy6aMsrSc4NGKSFWJeTJf9fNdsdD0TjvGcPSQeNHVOR9oxwpjIZvqPLrvh6rnNIqLKyc+azx5Z0OAlSFRVbUPjtAaaWEWPUK81I4rWkcoCosjFJJBwGZU0JM3cp1jEY9M90OqDt41ROT22uKAYve4EMilVFVfrikRKO9o/1hHrBMbqxOKbUWVRZ13l0GDwsFLIQVvjp7PB68blbGN5UeXGGJGlQJ2XATE3dkTFbeaYN/hdEgtKPW868ACQ8LkN6LhVUInd1W3ukgILVTpZhrLqWUkLYPi9X3+3Cglc5LK2DJnhKyOqmZUcs8LDoKS66BZy4elhMZj9vFleSOnhAPWCY1VsHtkviMI9FnpPZhKe3GcRSwlAFGOyo7q4T8HjcPfNii9ykrac7Dv8IQAxajJlCjhW63LB0Sisbx3r6EKbKcG8YxRElXXFhKISXEW/NrPCwlobCUSIUQoF7PUFTmaihjgAcsuV1zprDomW5z/Tvm6mE5kWGVQmLAwnyEtTxtl66wmEsJkemWKCB8nHxYv6zZrjRBVcCDYwMR/nNYSugkGxQW0cNipLCw3ixD0Thm/OBF1Aa98HtdiMRlNFT5MMGiWbkUkSQJfo8L4ZjMFyFAnczt5JRQlcbDEnJwWbMY+FWUwIRmEVGlGIrGU1rI5zpHiFHPTbdiWTMLWPJPCeX6GicaLTUBvAego2eIr8OTGpMBS9CLfRhKKW0OmVJYKCVEDANGKSG7d928UijEUkL2KSwjRYWlVr9M0e9x4++S/p7+cAwHuof4OVwweZTj2rsXCr2FRS1hd+41YDfJUFRGLC47vA+L+pmZMdr5E5pF/B4X71EyaDBKoyrH2V+1QT2Fxb6UUK6B1IkGqxR6d283BiNxeFwSry5lfyNRYQmbUFh8LCVULn1YCGeSzXRr12C5RN57CP3hGIYicRzoTnhJ8ulyyxBNt9outyKP3HA2jg1E0BuKoWcoip6hKELRONpPSu8IWa4EvG70hmIpCkssGbB4HKywiDv/gXBcLWt24K5a/MyUUjoISKhwlT4P+sOxtDWhP9+UUFIJDcdkDEXiCPrcNisszgtenQhTod/YcQQAMKGhkqurugGLBYUlEqeAhSggbIiZdpy8nY3jALHKI4ZdRxLKRl2FN685QowGIWDJNMhMkiSMrPKnKDInGnottFnFkB0G60Lh87h4OqsvHBUUFueds9ij5IwSMtwygj43+sMxDEQMFJYclYxKnxsel4SYrKB7KIKgL8hVHHs8LHRLMgNTWNh4gsmNqsqdSWEx1Zo/6lwPi/NWCsIy3MMSiaX05rCzSghIba3OW/I3VNqSimHt+YFEiTdhjNo8Tl1YSiElBKSOeBjKc2deSMQ06uljS69cnk9sNlBYcg0MJElSm8clS5u5wpKjapNSJUQKiym0m7pJQsBSoxOwhLjaXtpVQs5bKQjLsA+8oiTebGzHYmeVEKAucn3hGB+8ZYd/BQAaaxK9VzwuCU21J656YoaAjsISSyosTk4JAYn30JH+CPpDasCindbrBBqr/ThvUgNGVPpKYkKzlqBB9+t8TbdAolLoSH+YzxNiP6PCa0NKiBQWU2g3dZOyKCwhEwpLKfRhoXdHGSDuUAcj8bSAxbYqISElxBSWfFryi1T4PPifr58Dj1vKqWX4iQTfCYlVQjaraYWiSpgnNMgaxznQdOtySXj8ptnFPo2cqTTofs0UluocG8cBarfb7iHWsTi/7rmpjePolmSGxprUTZ24DuftYdEpa/6kqw/feeo9jB9ZifuvPSP3E88TeneUAW6XBJ/HhUhMxmAkxj0ldlcJiXK+WiGUv+GWcSIZZ/NBT2GJlkhKSAxYQg5uzV/qBHmrA3tTQoA4TyhVYcm5Nb9PXZ+oD4s5Al43Rlb6cHQgAklKDVjqgom/T0+ykisaV/vx5Nqa/0hfGO/t79HtnjycOHs7RpimUqdSyO4qIbb76QtFeZdbuxQWwjx+nfLDaImkhKoFH9RgNL8ZNIQx7LM6GLU/JVSnKW1mZc25qiNkus0NlqocXRdM+QxpFRYxAMm1020fL4cv7t/H2asbYRq9brd2VwmxN+unhwdsG3pIWIcpLKFYekrIyY3jgFSFZSiSOGcn9mEpdXh7foM+LPmkhFhpM/Ow5Nvplsqac4MZb8UKISA9YBHN+ZlSxtzDolMlxHpvVefYv8cunL26EaZRe7GoCxQbiGdbSih5s9l6INEKf5wNQw8J6+grLMzD4uyUUKXQnt/JnW5LHVaxo5XwB2xJCSUHIDKFJZxftZf4PGocZx42vf7kptQuzEYBi8/jgitDA8RMfVj6QonXqi7y34feHWVChU4Zo91lzWyRY4ugHS35CetwhUW3rNnZAaQ4ADHfDqmEMarimqqw9OXZhwVQPRI8JZRM7eVa1uxP9ueJxGV+syWyc+N5E+Fxu3DDnAkpj7NrOBSNIxKTVcNtls2l2N9JUZSUdhX9ofyVOTuggKVMqND1sCT+32tXp1vNm9WukmbCGnq55lIpa2Y7tOMDEbC5fE4say519NcDmfvaqnOc1gyIAxCTKaFwfikhl0vCfdecjv5wnBt6ieyMG1mBOz4/Le3x6oAHkpRoc8E6gQPpE8i1+N1qe4xoXIHPowYsdgS6dkABS5mgt6Oyu/up9s1qR0t+wjp+HYWlVFJC7D10qC/MH6OUkP3oBSzicNR8qnFqNWXN7GfkU5I8f3pLzs8lUnG5JFT7PXx8SSjKSprNKSxAQrEV0/3kYSFsRW+ekP2zhFIXpJMaSWEpBnoKS8mYbpML3uFkwOJ1S44/51JEbwPDDLcBrysvJY5NbO4ejECWFXXEAilljoEFlT1DUYRjbPBh5r+PuLHVGm+Zh4WqhAhbYM239AIWu6uEGKSwFAc9D0uplDWzoXuH+xMBC6krhUFPYVHnCOW3S1YDlmhK2TQ1fXMOqvE2ws352RQWl0viQYu2tNmO6jI7cPbqRpiGGWJF063djeNEhaU2aM/QQ8I6mRUWp6eEEgvpseTQNtqVFwa9Ngf5TmpmsCqhmKzgUG9iRIckZb8hEsMHbx4nKCwBEx3Ejdrz85RQkT0s9A4rE4LCAERG2O5ZQsLN5aRR9gw9JKzDUnz6HhZnf6S1Kh0pLIWBD0QVfCu8aVyeu+SA182Dk4PdiYClwuum9cBBcIVlUPWw+E0ElGp7fo3CQh4Wwk5YSkivrNmuXbfH7eI3GKoQKh6sW2WqwlIqKSFNwEJphILATLVD0XSFxY7UDUsLHeweAkB/R6ehTmxW+x2ZUVjU9vz6HhZKCRG2oGe6Zbtuu0y3gLo7o5b8xUNvUSmVlJB2wQtSGqEgBL1sUGl6wGLHTYft4A8kAxaaAeQsxOZxbGNjSmHR2QwB1JqfsBltUzdAlfV8bvsWE5bDtHPoIWENprCEdDvdOvsjre2wmmt3VCIzXGGJ6KSEbPAhaBUW+js6CzFgsaKwcNOtsLbIskKmW8JeKnRb8yd33R77dt1fnzMBF5w8CudPbrDtNQlr6E1VZSkhu5oEFoqE10H9d6ZhbETucMU1GoeiJN4bdkxqZjDj7cEeFrDQ39FJ6CssJlJCXtaeX934Jt5Dif/Pp+GgHVBYXCYENWXNsqwgJtvbOA4ArmufgOvaJ9j2eoR1uIdFx3TryTArxAm4XBKqfB4uMdONrjAwn4qiJJS4oM+tVgnZsEtmHWkPHKeAxYmwgKVX7HRrYjPDN0OCwsL8Kx6XVPRKMGdvxwjTMEmWmW7FAVY0oLC80FdYSqNxHJB6w6QqocIgXlemuvbbWJpazxWWZJUQBSyOIjUlxPqwmDHdpntYxOqyYleCOX91I0zBp7MmB5GJAUsp3MQI86gelvTGcaUQnIoeCurDUhhcLilNde2P2J8SYmlnahrnLFjA0j0UUfuwmFBHfDqG/l6HDD4EKGApG7TTmqNChOx0IyZhjUwKi9NTQkDqDZMClsKh7XZrp+lWO6SQ/o7OQk9hydaaP3FMeh8Wuzok2wHdycqECk0Zo9iDxVUCNzHCPPoKS+mkhKopJTQsMNWVNZO0s9KjLph687JDtSHsg80SCkVl9CY9KGYUFr3NkFN6sAAUsJQNbIczFI1DlhXb5wgRzoEtKjFZQSwZqJRqSoi8D4WDbWKY6jpgY5VQvWYsBwWezqLa7+HVeGwyujmFxdjDUuy2/AAFLGWD2LgpFIvbPkeIcA6ieS4SlyHLCuLJirBSSAmJAQuVNRcOrrAkA5U+W/uwaBUW+js6CZdLQg2bjJ6c92TNwyIqLORhIWxGbAo0GImrc4RIYSk7xCA0FJURlQWDdQkEqGKVECkshaNCUF0Buzvdpios1DjOeTAfC1NYzFUJpZtundLlFqCApWxIqQoIxyklVMa4XRJvwR+OxXk6CCiNADWlSogUloLBgoiBcKJ5nJ0poTqNwkKBp/NgAQvrx2WqD4vXuA9LsQcfAhSwlBW8KiAa4zcxO+cIEc6BKWqhqJxSEVYKASqVNQ8PYvfrcEzmNy47UkJet0vjRSr+7ptIpVZjjDajsLAxLrp9WMjDQtgJ78USURUW8rCUJ3wnFIvzlJAkJdQXp0ON44YHFkQMRuI8HQTY1zNFVFlIYXEe2oDFisIS0fGw1FBKiLATsSqAzYIohR03YR2/qLCwOUIl8rcmhWV4EPuwsF1ypc9tW5uDeqEXC5lunUdNDgqLnofFzpEO+VL8MyBsgy3+A+EYkuovKSxlipprjvOUUCn4V4BU0yftzAtHpZASKsRNR1RYgl66lTiN9JSQmT4s6Skh7mEptcZxy5Ytw9lnn43q6mo0NjZiwYIF2LZtW8bn/OY3v8H555+P+vp61NfXY+7cuXjrrbdSjrnhhhsgSVLK1/z5863/Nic4YlUAaxxXKjcxwhrcwxKTEUumhDxu56eDgNSUBJU1F46gTkrIzgZvdaSwOBqtMdpMHxbdsmYHKSyW7mZr167FokWLsH79erz00kuIRqO49NJLMTAwYPicNWvW4Nprr8Urr7yCdevWYezYsbj00ktx4MCBlOPmz5+Pjo4O/vXkk0/m9hudwIgSMK8SIoWlLBEVlkisxFJCATJrDgeVfkFhKUDzL7HbLaX2nEeah8VCp9uIWNbsoD4sls7ghRdeSPn3Y489hsbGRmzatAkXXHCB7nOeeOKJlH//9re/xR//+EesXr0aCxcu5I/7/X40NzdbOR1Cg2iyY5sfUljKE1FhiZaYmiZKy2S6LRzi8MNCpITE5nE0/NB55FIlpNeaXw12SywlpKWnpwcAMGLECNPPGRwcRDQaTXvOmjVr0NjYiClTpuCWW27B0aNHDV8jHA6jt7c35YsQFJZwjCssVNZcnogKS6mlhEZU+eBzu1Ad8JjKqxO5wdI/g2EhJWRjYCGmhCjwdB65VQklPSxRNvJD5o0HS05hEZFlGUuWLMGcOXMwffp008/77ne/i9bWVsydO5c/Nn/+fHzxi19EW1sbPv30U3zve9/DZZddhnXr1sHtTv8gLFu2DD/84Q9zPfWyJcj7sMR5NO0tkZsYYY2AYI4ruZSQ34PHvn42Aj43JInen4UiKPRlKqTpNui1r/KIsA8xYJEkcwosO4ZVCQ0I5fBO8LDkfAaLFi3C1q1b8frrr5t+zj333IMVK1ZgzZo1CAQC/PFrrrmG//+MGTMwc+ZMnHTSSVizZg0uueSStNdZunQpvv3tb/N/9/b2YuzYsTn+JuUDN91G4oj4qQ9LOcMUllA0XlKTmhnnTmoo9imUPUxNGQzHC+JhYWXNZLh1JmLA4ve4TG0OeB+W5JrC/CsBr8sR60tOZ7B48WKsXLkSr7zyCsaMGWPqOT/72c9wzz334MUXX8TMmTMzHjtx4kQ0NDRgx44dut/3+/2oqalJ+SJED0uMGseVOaLCogYstMslVFL6sBSgSqilLrHpbKjy2/aahH2IfVjMVuNxD0syJdTroLb8gEWFRVEU/OM//iOeeeYZrFmzBm1tbaaed++99+Luu+/GqlWrcNZZZ2U9fv/+/Th69ChaWlqsnN4JT0qVUAnuugnzpPRhKbHGccTwwNaDgQL1YZnaXINffOU0TG6stu01Cfuo9nsgSYCipA7HzYS2D0shlLl8sLTCLVq0CI8//jiWL1+O6upqdHZ2orOzE0NDQ/yYhQsXYunSpfzfP/nJT3DHHXfgkUcewYQJE/hz+vv7AQD9/f34zne+g/Xr12P37t1YvXo1rrrqKkyaNAnz5s2z6dc8MRADligpLGWN6OYnhYXQgymuQ5HCpIQA4AtnjMH00bW2viZhDy6XhJqkMmKmpBkQy5pTU0JOMNwCFgOWBx54AD09PbjooovQ0tLCv37/+9/zY/bu3YuOjo6U50QiEXz5y19Oec7PfvYzAIDb7cb777+PK6+8EieffDJuvPFGzJo1C6+99hr8fpIarRAUU0LJm5ifdt1lCZN4S9XDQhQeNlssJis4NhgBYG9KiHA+zMdiXmFRTbeKojiqLT+QQ0ooG2vWrEn59+7duzMeHwwGsWrVKiunQRhQKZpuY3QTK2dEhSVGKSFChwrBt3C4LwzAGRN3ieGjrsKLvcesKCyJ94ysJAJdJ7XlB2j4YVkR1PGwUEqoPBEVlgilhAgdPG4X//wf6g0BcM5OmRgeLCssQmATjsmOassPUMBSVoidbqlKqLzR97DQ35pIRTXeJvpqkMJyYsEqhcwqLGKvlkhMLm0PC+FsKoTprJQSKm94R0pKCREZ0Ha2pYDlxIIpLGYGHwIJoy5TasOxwpm1c4VWuDIipUqIUkJlDVNYKCVEZEI7lNAp0j4xPPCUkIURGLy0OSqrHhaH9GGhu1kZwVJC4Zg6/4FmCZUnAa9e4zj6WxOpVGoDFofslInhYXRdEIC15n4+Id1c0lVChLOpEBannqHEG41uYuWJqLBQwEIYISosLomGFJ5ofHnWGAS9blw0ZZTp54i9WHod5mFxxlkQtpCYF5HobNiT7LtgZuAVUXoEdD0slBIiUhE9LJV+Dw2bPMEIeN340ixz43MYYi8W5mFxijJHd7MyQpIkvkD1DCVyj+RhKU/0PSz0tyZSERUWpxgnCWcjtufvC5OHhSggQU0ZI93EyhO/7vBD+lsTqYhpYqf4EAhn49NRWJySEqIVrsyo0JjsSGEpT5jrPxSNU0qIMKRCkxIiiGyIE5upDwtRULSmOqoSKk/EPiyUEiKMSFFYKGAhTMCazPWGoojJic2QU947tMKVGdpdFN3EypOA4OSnJoGEEeJ64JRdMuFsWLr5SH+icEOS0hsQFgta4coMSgmdGPgFJW0gzErYKSVEpCIqrk656RDOhlWWHk0GLFU+D1wuZ6wtdDcrM7QpIQpYypOA8HftD1PPHUKfSj+ZbglrsJTQkf7EhG8nKXO0wpUZWoWFdt3licftgju562HGOApYCC1BQVWhsmbCDMz3eHSABSzOKGkGKGApOyo0i5LfTZ0tyxWmsrCAxUPBKaFBbM1PVUKEGZiHhaeESGEhCkUFpYROGJiPhaWEqKsxoSVIfVgIi7B7BjPdUkqIKBiUEjpxYAoLeVgII0SjrVNKUwlnw1JCx5IpISe9b2iFKzPEnLVLSngdiPKEKSzxZK8EL6lphAbqw0JYhaWEkssKeViIwiEuUJQOKm+0TQG9Dik9JJyD6GmjgIUwA6sSYlBKiCgYYsBCKYLyxq/xK5HCQmgRPW1kuiXMoPXCOam6jFa4MkOcHUJt+cubNIWFAlRCQ4XQh8VJO2XCuWgVFieZtWmFKzNSUkJ0AytrAhqFxUMpIUKDz+1CS20AlT43RlX7i306RAnAPCwMJ3lYnBM6EbYgljFSiqC80Sos5FkitEiShGcXzUEoGk9RXwnCCO064iTvk3POhLAFsYyRFJbyRquwUEqI0KOpJlDsUyBKCO1GqIZSQkShCFKV0AmDdmGhlBBBEPmiXVfIw0IUDKoSOnEIeCklRBCEvTjZw0IrXJlBfVhOHLQLCwWoBEHki5M9LLTClRlU1nzioFVYaAwDQRD5or1vOKkcnu5oZYbP4+JeBtpxlzeksBAEYTfiRsjrlhy18XXOmRC2wYy3VCVU3qQrLPT3JggiP8SNUHXAC0lyjnJLK1wZwnws5GEpb8SFxSUBbqoSIggiT8T7hpP8KwAFLGUJ68VCO+7yRlRYaCo3QRB2IKaAnORfAShgKUuCpLCcEIgKC6X/CIKwA3FdIYWFKDgsJeQksxRhP+LflyqECIKwA1+KwuKcHiwABSxlSZCnhOgmVs6IrfkpJUQQhB24XRKvNHVSW36AApaypJJSQicEosJCKSGCIOyCrS1OassPUMBSlqhlze4sRxKljN8rjmEgNY0gCHtgawuZbomCc96kBlT5PTh7Qn2xT4UoIKLCQikhgiDsgissfmd5WJwVPhG28MUzx2DB6aPhor4cZU3AS4MuCYKwH2YnIIWFGBYoWCl/Uj0s9PcmCMIe/OUQsCxbtgxnn302qqur0djYiAULFmDbtm1Zn/fUU09h6tSpCAQCmDFjBp577rmU7yuKgjvvvBMtLS0IBoOYO3cutm/fbu03IYgTDFJYCIIoBLXBRCpoZKW/yGeSiqVVbu3atVi0aBHWr1+Pl156CdFoFJdeeikGBgYMn/Pmm2/i2muvxY033oh3330XCxYswIIFC7B161Z+zL333ov7778fDz74IDZs2IDKykrMmzcPoVAo99+MIMocf0qnW1JYCIKwh3+7fBq+O38q2k8aWexTSUFSFEXJ9cmHDx9GY2Mj1q5diwsuuED3mK985SsYGBjAypUr+WOf+cxncPrpp+PBBx+EoihobW3FP//zP+Nf/uVfAAA9PT1oamrCY489hmuuuSbrefT29qK2thY9PT2oqanJ9dchiJIiEpNx8u3PAwDOn9yA/3fj7CKfEUEQhDWs3L/z0pF7enoAACNGjDA8Zt26dZg7d27KY/PmzcO6desAALt27UJnZ2fKMbW1tZg9ezY/Rks4HEZvb2/KF0GcaHjdEtggVerDQhBEuZPzKifLMpYsWYI5c+Zg+vTphsd1dnaiqakp5bGmpiZ0dnby77PHjI7RsmzZMtTW1vKvsWPH5vprEETJIkkSAsm5H5QSIgii3Mk5YFm0aBG2bt2KFStW2Hk+pli6dCl6enr41759+4b9HAjCCTAfC5luCYIod3KqWVq8eDFWrlyJV199FWPGjMl4bHNzM7q6ulIe6+rqQnNzM/8+e6ylpSXlmNNPP133Nf1+P/x+Z7mXCaIYsPJDSgkRBFHuWFrlFEXB4sWL8cwzz+Dll19GW1tb1ue0t7dj9erVKY+99NJLaG9vBwC0tbWhubk55Zje3l5s2LCBH0MQhD6stJlSQgRBlDuWFJZFixZh+fLl+NOf/oTq6mruMamtrUUwGAQALFy4EKNHj8ayZcsAALfeeisuvPBC/PznP8fll1+OFStWYOPGjXjooYcAJPLwS5YswY9//GNMnjwZbW1tuOOOO9Da2ooFCxbY+KsSRPnBFBZKCREEUe5YClgeeOABAMBFF12U8vijjz6KG264AQCwd+9euFzq4nnuuedi+fLluP322/G9730PkydPxrPPPpti1L3tttswMDCAm2++Gd3d3TjvvPPwwgsvIBAI5PhrEcSJAVNYKGAhCKLcyasPi1OgPizEicrVD76Jt3cfx80XTMT3PndKsU+HIAjCEsPWh4UgiOLCPSw0O4ogiDKHAhaCKGHIw0IQxIkCrXIEUcL4kwqLz0MfZYIgyhta5QiihBlZ6QOgTlclCIIoV3JqHEcQhDNY9HeTMLmxCl84Y3SxT4UgCKKgUMBCECVMU00A17VPKPZpEARBFBxKCREEQRAE4XgoYCEIgiAIwvFQwEIQBEEQhOOhgIUgCIIgCMdDAQtBEARBEI6HAhaCIAiCIBwPBSwEQRAEQTgeClgIgiAIgnA8FLAQBEEQBOF4KGAhCIIgCMLxUMBCEARBEITjoYCFIAiCIAjHQwELQRAEQRCOpyymNSuKAgDo7e0t8pkQBEEQBGEWdt9m9/FMlEXA0tfXBwAYO3Zskc+EIAiCIAir9PX1oba2NuMxkmImrHE4sizj4MGDqK6uhiRJtr52b28vxo4di3379qGmpsbW1yZSoWs9fNC1Hj7oWg8fdK2HD7uutaIo6OvrQ2trK1yuzC6VslBYXC4XxowZU9CfUVNTQx+AYYKu9fBB13r4oGs9fNC1Hj7suNbZlBUGmW4JgiAIgnA8FLAQBEEQBOF4KGDJgt/vx/e//334/f5in0rZQ9d6+KBrPXzQtR4+6FoPH8W41mVhuiUIgiAIorwhhYUgCIIgCMdDAQtBEARBEI6HAhaCIAiCIBwPBSwEQRAEQTgeClgy8Ktf/QoTJkxAIBDA7Nmz8dZbbxX7lEqeZcuW4eyzz0Z1dTUaGxuxYMECbNu2LeWYUCiERYsWYeTIkaiqqsKXvvQldHV1FemMy4d77rkHkiRhyZIl/DG61vZx4MABfO1rX8PIkSMRDAYxY8YMbNy4kX9fURTceeedaGlpQTAYxNy5c7F9+/YinnFpEo/Hcccdd6CtrQ3BYBAnnXQSfvSjH6XMoqFrnTuvvvoqrrjiCrS2tkKSJDz77LMp3zdzbY8dO4avfvWrqKmpQV1dHW688Ub09/fnf3IKocuKFSsUn8+nPPLII8oHH3ygfPOb31Tq6uqUrq6uYp9aSTNv3jzl0UcfVbZu3aps3rxZ+dznPqeMGzdO6e/v58d861vfUsaOHausXr1a2bhxo/KZz3xGOffcc4t41qXPW2+9pUyYMEGZOXOmcuutt/LH6Vrbw7Fjx5Tx48crN9xwg7JhwwZl586dyqpVq5QdO3bwY+655x6ltrZWefbZZ5X33ntPufLKK5W2tjZlaGioiGdeetx9993KyJEjlZUrVyq7du1SnnrqKaWqqkq57777+DF0rXPnueeeU/7t3/5NefrppxUAyjPPPJPyfTPXdv78+cppp52mrF+/XnnttdeUSZMmKddee23e50YBiwHnnHOOsmjRIv7veDyutLa2KsuWLSviWZUfhw4dUgAoa9euVRRFUbq7uxWv16s89dRT/JiPPvpIAaCsW7euWKdZ0vT19SmTJ09WXnrpJeXCCy/kAQtda/v47ne/q5x33nmG35dlWWlublZ++tOf8se6u7sVv9+vPPnkk8NximXD5ZdfrnzjG99IeeyLX/yi8tWvflVRFLrWdqINWMxc2w8//FABoLz99tv8mOeff16RJEk5cOBAXudDKSEdIpEINm3ahLlz5/LHXC4X5s6di3Xr1hXxzMqPnp4eAMCIESMAAJs2bUI0Gk259lOnTsW4cePo2ufIokWLcPnll6dcU4CutZ38+c9/xllnnYWrr74ajY2NOOOMM/Cb3/yGf3/Xrl3o7OxMuda1tbWYPXs2XWuLnHvuuVi9ejU++eQTAMB7772H119/HZdddhkAutaFxMy1XbduHerq6nDWWWfxY+bOnQuXy4UNGzbk9fPLYvih3Rw5cgTxeBxNTU0pjzc1NeHjjz8u0lmVH7IsY8mSJZgzZw6mT58OAOjs7ITP50NdXV3KsU1NTejs7CzCWZY2K1aswDvvvIO333477Xt0re1j586deOCBB/Dtb38b3/ve9/D222/jn/7pn+Dz+XD99dfz66m3ptC1tsa//uu/ore3F1OnToXb7UY8Hsfdd9+Nr371qwBA17qAmLm2nZ2daGxsTPm+x+PBiBEj8r7+FLAQRWPRokXYunUrXn/99WKfSlmyb98+3HrrrXjppZcQCASKfTpljSzLOOuss/Dv//7vAIAzzjgDW7duxYMPPojrr7++yGdXXvzv//4vnnjiCSxfvhynnnoqNm/ejCVLlqC1tZWudZlDKSEdGhoa4Ha706olurq60NzcXKSzKi8WL16MlStX4pVXXsGYMWP4483NzYhEIuju7k45nq69dTZt2oRDhw7hzDPPhMfjgcfjwdq1a3H//ffD4/GgqamJrrVNtLS0YNq0aSmPnXLKKdi7dy8A8OtJa0r+fOc738G//uu/4pprrsGMGTNw3XXX4f/+3/+LZcuWAaBrXUjMXNvm5mYcOnQo5fuxWAzHjh3L+/pTwKKDz+fDrFmzsHr1av6YLMtYvXo12tvbi3hmpY+iKFi8eDGeeeYZvPzyy2hra0v5/qxZs+D1elOu/bZt27B371669ha55JJLsGXLFmzevJl/nXXWWfjqV7/K/5+utT3MmTMnrTz/k08+wfjx4wEAbW1taG5uTrnWvb292LBhA11riwwODsLlSr11ud1uyLIMgK51ITFzbdvb29Hd3Y1NmzbxY15++WXIsozZs2fndwJ5WXbLmBUrVih+v1957LHHlA8//FC5+eablbq6OqWzs7PYp1bS3HLLLUptba2yZs0apaOjg38NDg7yY771rW8p48aNU15++WVl48aNSnt7u9Le3l7Esy4fxCohRaFrbRdvvfWW4vF4lLvvvlvZvn278sQTTygVFRXK448/zo+55557lLq6OuVPf/qT8v777ytXXXUVldrmwPXXX6+MHj2alzU//fTTSkNDg3LbbbfxY+ha505fX5/y7rvvKu+++64CQPmP//gP5d1331X27NmjKIq5azt//nzljDPOUDZs2KC8/vrryuTJk6msudD853/+pzJu3DjF5/Mp55xzjrJ+/fpin1LJA0D369FHH+XHDA0NKf/wD/+g1NfXKxUVFcoXvvAFpaOjo3gnXUZoAxa61vbxl7/8RZk+fbri9/uVqVOnKg899FDK92VZVu644w6lqalJ8fv9yiWXXKJs27atSGdbuvT29iq33nqrMm7cOCUQCCgTJ05U/u3f/k0Jh8P8GLrWufPKK6/ortHXX3+9oijmru3Ro0eVa6+9VqmqqlJqamqUr3/960pfX1/e5yYpitAekCAIgiAIwoGQh4UgCIIgCMdDAQtBEARBEI6HAhaCIAiCIBwPBSwEQRAEQTgeClgIgiAIgnA8FLAQBEEQBOF4KGAhCIIgCMLxUMBCEARBEITjoYCFIAiCIAjHQwELQRAEQRCOhwIWgiAIgiAcDwUsBEEQBEE4nv8fixogb98tKbIAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_iterator = c4_data_loader(8, seq_length)\n",
    "train_state = init_fn()\n",
    "\n",
    "losses = []\n",
    "for _, batch in zip(trange(100), data_iterator):\n",
    "    train_state, loss = train_step(train_state, batch)\n",
    "    losses.append(loss)\n",
    "\n",
    "plt.plot(losses)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.5"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "15c8c58f52904c42ac3decf3f512c155": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_ac90f87da15544239e4583f57c5be437",
       "IPY_MODEL_c2259bb6aa6640a49746f3372eb4c4c0",
       "IPY_MODEL_2c28c4038cb74a37be8b87569815473a"
      ],
      "layout": "IPY_MODEL_2dca7026881b4c76afc714780b0fe912"
     }
    },
    "173b1a6064104ec68e8bed5e737fcff7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9a9e2cf126ca42768d0a3620dae27efc",
      "placeholder": "​",
      "style": "IPY_MODEL_a4c98eafd67f4cbab889205378f034f3",
      "value": " 1648877/1648877 [00:00&lt;00:00, 37536385.77it/s]"
     }
    },
    "174ce49279334896b7a55c6b73d40742": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_2effb9066b6242d083e9c8912a08e0ed",
      "max": 9912422,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_2381816cbc684abb8769f9cc6a3c36a6",
      "value": 9912422
     }
    },
    "1c351553e7e54cfbb4012f852ee23ecc": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "2381816cbc684abb8769f9cc6a3c36a6": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "246fb8fda8aa43c69ce8244ecbbe4edc": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "24f03b4702a144caba138246b4a4c5ac": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_5bb2b47586a3427d9c7a345e92b133f4",
       "IPY_MODEL_174ce49279334896b7a55c6b73d40742",
       "IPY_MODEL_55a0f4c81df0474f8ee98f90610a714e"
      ],
      "layout": "IPY_MODEL_7be2836bf0f04a6ea3f1a2844e9342a4"
     }
    },
    "2c28c4038cb74a37be8b87569815473a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_84bb132cb6f84c558f4ff8f5962e75a0",
      "placeholder": "​",
      "style": "IPY_MODEL_6c534a02f0074a3ab059327b6c4dbdc8",
      "value": " 28881/28881 [00:00&lt;00:00, 1105706.67it/s]"
     }
    },
    "2dca7026881b4c76afc714780b0fe912": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2dfe05e522ee446c8fdfc057db6b8d43": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2effb9066b6242d083e9c8912a08e0ed": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "33263e377ba24465a881e1ad401eaa7f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "38b4b5b1ef8645d5b3c0ed99e0326ed9": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "38d68ead3f8646818cb3918f1a9cf226": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_4a6146bf153146ec9aba6a6ae7684bda",
      "max": 1648877,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_a545f261b04742f1a267ba3489197504",
      "value": 1648877
     }
    },
    "3a62ce77e2ed437e8316e28b0498f561": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3d778be8c7d34a8ab58ffb1092a67e35": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "43f353406f6d4860919dd7640f9d8c47": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_5515805cdb7641e8860aadc79fff4f35",
       "IPY_MODEL_4e0236311f604315bc8957e8ba7556b7",
       "IPY_MODEL_9473b8d5cb9745af8f447fd3980eb1bf"
      ],
      "layout": "IPY_MODEL_b0431de03fe5488689290692f125d937"
     }
    },
    "4a6146bf153146ec9aba6a6ae7684bda": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4e0236311f604315bc8957e8ba7556b7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "danger",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3a62ce77e2ed437e8316e28b0498f561",
      "max": 10,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_5d66b9402ff345079709abba99df6c39",
      "value": 0
     }
    },
    "5515805cdb7641e8860aadc79fff4f35": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_995b5be773434a5b90dc442dfe96e6f5",
      "placeholder": "​",
      "style": "IPY_MODEL_6084b93b68984bdab32ec8eef8fe4f63",
      "value": "  0%"
     }
    },
    "55a0f4c81df0474f8ee98f90610a714e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_61c4402bd30348069b1a9ddbaa01ca0d",
      "placeholder": "​",
      "style": "IPY_MODEL_f06e1694702e4b2c89c4c1e461d2751b",
      "value": " 9912422/9912422 [00:00&lt;00:00, 109457870.29it/s]"
     }
    },
    "57164d4c93884b07a4bf8b32fa7f0039": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5bb2b47586a3427d9c7a345e92b133f4": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f54ae867983645dfbad1b8fcd3168c1d",
      "placeholder": "​",
      "style": "IPY_MODEL_1c351553e7e54cfbb4012f852ee23ecc",
      "value": "100%"
     }
    },
    "5d66b9402ff345079709abba99df6c39": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "6084b93b68984bdab32ec8eef8fe4f63": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "61c4402bd30348069b1a9ddbaa01ca0d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6a37de50dbd244939e2862ca4ec8ec95": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_c6462f0ffb21428ab978dbebfcff39f9",
      "max": 4542,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_38b4b5b1ef8645d5b3c0ed99e0326ed9",
      "value": 4542
     }
    },
    "6c534a02f0074a3ab059327b6c4dbdc8": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "7495ca783bb8425e8f7b7f5b2e884edb": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7895078f940b476291a4de1d17670d72": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "7be2836bf0f04a6ea3f1a2844e9342a4": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7ef1e63f445d4014be6705105bc8c4f1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_d3a988c9b68848feb76dff109ee83f9d",
       "IPY_MODEL_6a37de50dbd244939e2862ca4ec8ec95",
       "IPY_MODEL_9ccdaca10125485a9b7a1a4dc45e0b7c"
      ],
      "layout": "IPY_MODEL_2dfe05e522ee446c8fdfc057db6b8d43"
     }
    },
    "840c9c7c74fd430ca79335a1427c4f7e": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "84bb132cb6f84c558f4ff8f5962e75a0": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "908917c27c7c407a8a406077d035b150": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "92bbd02596564093919add1bcc6bfecd": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b682c0ed6cd94886b1a5eb3e9c37eb47",
      "placeholder": "​",
      "style": "IPY_MODEL_a6324a250b864485b74eb801b934a88f",
      "value": "100%"
     }
    },
    "9473b8d5cb9745af8f447fd3980eb1bf": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_dac82a2c233c4ad38964e17f1daedf12",
      "placeholder": "​",
      "style": "IPY_MODEL_beaaa4def1da4ffba57aa220b4acb1c7",
      "value": " 0/10 [00:03&lt;?, ?it/s]"
     }
    },
    "94c68f786489421993d1aecb5dbe9456": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b7973aa4b5ef44e6b438de004b8b364b",
      "placeholder": "​",
      "style": "IPY_MODEL_33263e377ba24465a881e1ad401eaa7f",
      "value": " 14/59 [00:03&lt;00:07,  5.63it/s]"
     }
    },
    "995b5be773434a5b90dc442dfe96e6f5": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9a9e2cf126ca42768d0a3620dae27efc": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9b09d5fc4ee1409481dea26d27121fab": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "9ccdaca10125485a9b7a1a4dc45e0b7c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_bfd9d6d6c869472e82bbc8521b9e805a",
      "placeholder": "​",
      "style": "IPY_MODEL_9b09d5fc4ee1409481dea26d27121fab",
      "value": " 4542/4542 [00:00&lt;00:00, 159752.86it/s]"
     }
    },
    "a4c98eafd67f4cbab889205378f034f3": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "a545f261b04742f1a267ba3489197504": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "a6324a250b864485b74eb801b934a88f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "ac90f87da15544239e4583f57c5be437": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_eb13a3c1e21c4998aff32b220febc8df",
      "placeholder": "​",
      "style": "IPY_MODEL_246fb8fda8aa43c69ce8244ecbbe4edc",
      "value": "100%"
     }
    },
    "b0431de03fe5488689290692f125d937": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b682c0ed6cd94886b1a5eb3e9c37eb47": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b7973aa4b5ef44e6b438de004b8b364b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "beaaa4def1da4ffba57aa220b4acb1c7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "bf90a6df4fad48ba880c0f44da7ffd0e": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "bfd9d6d6c869472e82bbc8521b9e805a": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c2259bb6aa6640a49746f3372eb4c4c0": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_840c9c7c74fd430ca79335a1427c4f7e",
      "max": 28881,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_7895078f940b476291a4de1d17670d72",
      "value": 28881
     }
    },
    "c6462f0ffb21428ab978dbebfcff39f9": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d263f2f985d44b22b13ccfb8649f8c80": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "danger",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_bf90a6df4fad48ba880c0f44da7ffd0e",
      "max": 59,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_fe3a9a90f7544f278c2c97283e0d7636",
      "value": 14
     }
    },
    "d3a988c9b68848feb76dff109ee83f9d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_908917c27c7c407a8a406077d035b150",
      "placeholder": "​",
      "style": "IPY_MODEL_3d778be8c7d34a8ab58ffb1092a67e35",
      "value": "100%"
     }
    },
    "d7e7587334f74e5194f94b23d68487f1": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "dac82a2c233c4ad38964e17f1daedf12": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "eb13a3c1e21c4998aff32b220febc8df": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ee71e23f6ec14bb2a1ae77316a2de5c2": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d7e7587334f74e5194f94b23d68487f1",
      "placeholder": "​",
      "style": "IPY_MODEL_f0b0c371a4174a0ab13d7eda2b45cf5b",
      "value": " 24%"
     }
    },
    "f06e1694702e4b2c89c4c1e461d2751b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "f0b0c371a4174a0ab13d7eda2b45cf5b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "f123a6e3add04243991f57b6f91b0e0d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_92bbd02596564093919add1bcc6bfecd",
       "IPY_MODEL_38d68ead3f8646818cb3918f1a9cf226",
       "IPY_MODEL_173b1a6064104ec68e8bed5e737fcff7"
      ],
      "layout": "IPY_MODEL_57164d4c93884b07a4bf8b32fa7f0039"
     }
    },
    "f54ae867983645dfbad1b8fcd3168c1d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "fcec22af8df74d9c9a0215638dbfe595": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_ee71e23f6ec14bb2a1ae77316a2de5c2",
       "IPY_MODEL_d263f2f985d44b22b13ccfb8649f8c80",
       "IPY_MODEL_94c68f786489421993d1aecb5dbe9456"
      ],
      "layout": "IPY_MODEL_7495ca783bb8425e8f7b7f5b2e884edb"
     }
    },
    "fe3a9a90f7544f278c2c97283e0d7636": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}