ventilator-pressure-transformer-v2-Kaggle.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"V1: Add PositionalEncoding\n\nV2: Correct Transformer","metadata":{"id":"TRRDmcGpsh4u"}},{"cell_type":"code","source":"# Update pandas version for Colab TPU\n#!pip install pandas==1.3.2","metadata":{"id":"EPJR-B_AhTBC","execution":{"iopub.status.busy":"2021-10-14T04:32:29.162185Z","iopub.execute_input":"2021-10-14T04:32:29.162562Z","iopub.status.idle":"2021-10-14T04:32:29.185252Z","shell.execute_reply.started":"2021-10-14T04:32:29.162468Z","shell.execute_reply":"2021-10-14T04:32:29.184545Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"code","source":"try:\n  import google.colab\n  IN_COLAB = True\nexcept:\n  IN_COLAB = False","metadata":{"id":"mJaAgiAOkUEt","execution":{"iopub.status.busy":"2021-10-14T04:32:29.186886Z","iopub.execute_input":"2021-10-14T04:32:29.187318Z","iopub.status.idle":"2021-10-14T04:32:29.196993Z","shell.execute_reply.started":"2021-10-14T04:32:29.187286Z","shell.execute_reply":"2021-10-14T04:32:29.196113Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"if IN_COLAB:\n  from google.colab import drive\n  drive.mount(\"/gdrive\", force_remount=True)","metadata":{"id":"oH5W_yWm-P8P","outputId":"9b64a639-5c21-4d4c-b364-f840b7595786","execution":{"iopub.status.busy":"2021-10-14T04:32:29.198245Z","iopub.execute_input":"2021-10-14T04:32:29.198867Z","iopub.status.idle":"2021-10-14T04:32:29.207467Z","shell.execute_reply.started":"2021-10-14T04:32:29.198832Z","shell.execute_reply":"2021-10-14T04:32:29.206861Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"code","source":"if IN_COLAB:\n  !pip install kaggle\n  !mkdir /root/.kaggle\n  !cp /gdrive/MyDrive/ventilator-pressure-prediction/kaggle.json /root/.kaggle\n  !kaggle competitions download -c ventilator-pressure-prediction\n  !mkdir -p /kaggle/input/ventilator-pressure-prediction\n  !unzip '*.zip' -d /kaggle/input/ventilator-pressure-prediction\n  !ls /kaggle/input/ventilator-pressure-prediction","metadata":{"id":"mhjzjzp5etAG","outputId":"fa54ba87-8a32-488f-aba9-429826485429","execution":{"iopub.status.busy":"2021-10-14T04:32:29.208531Z","iopub.execute_input":"2021-10-14T04:32:29.209378Z","iopub.status.idle":"2021-10-14T04:32:29.226175Z","shell.execute_reply.started":"2021-10-14T04:32:29.209311Z","shell.execute_reply":"2021-10-14T04:32:29.225352Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\n# import optuna\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom tensorflow.keras.callbacks import LearningRateScheduler, ReduceLROnPlateau\nfrom tensorflow.keras.optimizers.schedules import ExponentialDecay\n\nfrom sklearn.metrics import mean_absolute_error as mae\nfrom sklearn.preprocessing import RobustScaler, normalize\nfrom sklearn.model_selection import train_test_split, GroupKFold, KFold\n\nfrom IPython.display import display\n\nmodel_folder = '/gdrive/MyDrive/ventilator-pressure-prediction/transformer/'\n\n\n","metadata":{"id":"a16844ae","execution":{"iopub.status.busy":"2021-10-14T04:32:29.228564Z","iopub.execute_input":"2021-10-14T04:32:29.228841Z","iopub.status.idle":"2021-10-14T04:32:35.141502Z","shell.execute_reply.started":"2021-10-14T04:32:29.228812Z","shell.execute_reply":"2021-10-14T04:32:35.140877Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"import os\nimport random\n# Deep Learning\nimport tensorflow as tf\nfrom tensorflow import keras\n# Metrics\nfrom sklearn.metrics import mean_absolute_error\n\n# Random Seed Initialize\nRANDOM_SEED = 42\n\ndef seed_everything(seed=RANDOM_SEED):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    random.seed(seed)\n    tf.random.set_seed(seed)\n    \nseed_everything()","metadata":{"id":"LEHQxSneTcVX","execution":{"iopub.status.busy":"2021-10-14T04:32:35.143078Z","iopub.execute_input":"2021-10-14T04:32:35.143563Z","iopub.status.idle":"2021-10-14T04:32:35.151702Z","shell.execute_reply.started":"2021-10-14T04:32:35.143518Z","shell.execute_reply":"2021-10-14T04:32:35.150718Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"code","source":"DEBUG = False\n\ntrain = pd.read_csv('/kaggle/input/ventilator-pressure-prediction/train.csv')\ntest = pd.read_csv('/kaggle/input/ventilator-pressure-prediction/test.csv')\nsubmission = pd.read_csv('/kaggle/input/ventilator-pressure-prediction/sample_submission.csv')\n\nif DEBUG:\n    train = train[:80*1000]","metadata":{"id":"564144b2","execution":{"iopub.status.busy":"2021-10-14T04:32:35.153332Z","iopub.execute_input":"2021-10-14T04:32:35.153667Z","iopub.status.idle":"2021-10-14T04:32:49.151768Z","shell.execute_reply.started":"2021-10-14T04:32:35.153631Z","shell.execute_reply":"2021-10-14T04:32:49.150964Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"code","source":"def add_features(df):\n    df['cross']= df['u_in'] * df['u_out']\n    df['cross2']= df['time_step'] * df['u_out']\n    df['area'] = df['time_step'] * df['u_in']\n    df['area'] = df.groupby('breath_id')['area'].cumsum()\n    df['u_in_cumsum'] = (df['u_in']).groupby(df['breath_id']).cumsum()\n    print(\"Step-1...Completed\")\n    \n    df['u_in_lag1'] = df.groupby('breath_id')['u_in'].shift(1)\n    df['u_out_lag1'] = df.groupby('breath_id')['u_out'].shift(1)\n    df['u_in_lag_back1'] = df.groupby('breath_id')['u_in'].shift(-1)\n    df['u_out_lag_back1'] = df.groupby('breath_id')['u_out'].shift(-1)\n    df['u_in_lag2'] = df.groupby('breath_id')['u_in'].shift(2)\n    df['u_out_lag2'] = df.groupby('breath_id')['u_out'].shift(2)\n    df['u_in_lag_back2'] = df.groupby('breath_id')['u_in'].shift(-2)\n    df['u_out_lag_back2'] = df.groupby('breath_id')['u_out'].shift(-2)\n    df['u_in_lag3'] = df.groupby('breath_id')['u_in'].shift(3)\n    df['u_out_lag3'] = df.groupby('breath_id')['u_out'].shift(3)\n    df['u_in_lag_back3'] = df.groupby('breath_id')['u_in'].shift(-3)\n    df['u_out_lag_back3'] = df.groupby('breath_id')['u_out'].shift(-3)\n    df['u_in_lag4'] = df.groupby('breath_id')['u_in'].shift(4)\n    df['u_out_lag4'] = df.groupby('breath_id')['u_out'].shift(4)\n    df['u_in_lag_back4'] = df.groupby('breath_id')['u_in'].shift(-4)\n    df['u_out_lag_back4'] = df.groupby('breath_id')['u_out'].shift(-4)\n    df = df.fillna(0)\n    print(\"Step-2...Completed\")\n    \n    df['breath_id__u_in__max'] = df.groupby(['breath_id'])['u_in'].transform('max')\n    df['breath_id__u_out__max'] = df.groupby(['breath_id'])['u_out'].transform('max')\n    df['breath_id__u_in__diffmax'] = df.groupby(['breath_id'])['u_in'].transform('max') - df['u_in']\n    df['breath_id__u_in__diffmean'] = df.groupby(['breath_id'])['u_in'].transform('mean') - df['u_in']\n    df['breath_id__u_in__diffmax'] = df.groupby(['breath_id'])['u_in'].transform('max') - df['u_in']\n    df['breath_id__u_in__diffmean'] = df.groupby(['breath_id'])['u_in'].transform('mean') - df['u_in']\n    print(\"Step-3...Completed\")\n    \n    df['u_in_diff1'] = df['u_in'] - df['u_in_lag1']\n    df['u_out_diff1'] = df['u_out'] - df['u_out_lag1']\n    df['u_in_diff2'] = df['u_in'] - df['u_in_lag2']\n    df['u_out_diff2'] = df['u_out'] - df['u_out_lag2']\n    df['u_in_diff3'] = df['u_in'] - df['u_in_lag3']\n    df['u_out_diff3'] = df['u_out'] - df['u_out_lag3']\n    df['u_in_diff4'] = df['u_in'] - df['u_in_lag4']\n    df['u_out_diff4'] = df['u_out'] - df['u_out_lag4']\n    print(\"Step-4...Completed\")\n    \n    df['one'] = 1\n    df['count'] = (df['one']).groupby(df['breath_id']).cumsum()\n    df['u_in_cummean'] =df['u_in_cumsum'] /df['count']\n    \n    df['breath_id_lag']=df['breath_id'].shift(1).fillna(0)\n    df['breath_id_lag2']=df['breath_id'].shift(2).fillna(0)\n    df['breath_id_lagsame']=np.select([df['breath_id_lag']==df['breath_id']],[1],0)\n    df['breath_id_lag2same']=np.select([df['breath_id_lag2']==df['breath_id']],[1],0)\n    df['breath_id__u_in_lag'] = df['u_in'].shift(1).fillna(0)\n    df['breath_id__u_in_lag'] = df['breath_id__u_in_lag'] * df['breath_id_lagsame']\n    df['breath_id__u_in_lag2'] = df['u_in'].shift(2).fillna(0)\n    df['breath_id__u_in_lag2'] = df['breath_id__u_in_lag2'] * df['breath_id_lag2same']\n    \n    df['R'] = df['R'].astype(str)\n    df['C'] = df['C'].astype(str)\n    df['R__C'] = df[\"R\"].astype(str) + '__' + df[\"C\"].astype(str)\n    df = pd.get_dummies(df)\n    print(\"Step-5...Completed\")\n    \n    return df\n\n\ntrain = add_features(train)\ntest = add_features(test)","metadata":{"id":"08fd8094","outputId":"74824d59-d6db-49f8-9cc4-785ff0eb68e0","execution":{"iopub.status.busy":"2021-10-14T04:32:49.153394Z","iopub.execute_input":"2021-10-14T04:32:49.153928Z","iopub.status.idle":"2021-10-14T04:33:42.631534Z","shell.execute_reply.started":"2021-10-14T04:32:49.153888Z","shell.execute_reply":"2021-10-14T04:33:42.630610Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"id":"7d9dcaa1","outputId":"d057b29e-ceef-4851-c436-13c462c2b394","execution":{"iopub.status.busy":"2021-10-14T04:33:42.633048Z","iopub.execute_input":"2021-10-14T04:33:42.633386Z","iopub.status.idle":"2021-10-14T04:33:42.665375Z","shell.execute_reply.started":"2021-10-14T04:33:42.633342Z","shell.execute_reply":"2021-10-14T04:33:42.664487Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"id":"e2da0be1","outputId":"76cbb467-bd42-4aee-f9a1-0c51389d1288","execution":{"iopub.status.busy":"2021-10-14T04:33:42.666696Z","iopub.execute_input":"2021-10-14T04:33:42.666939Z","iopub.status.idle":"2021-10-14T04:33:42.673278Z","shell.execute_reply.started":"2021-10-14T04:33:42.666912Z","shell.execute_reply":"2021-10-14T04:33:42.672169Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"targets = train[['pressure']].to_numpy().reshape(-1, 80)\n\ntrain.drop(['pressure','id', 'breath_id','one','count',\n            'breath_id_lag','breath_id_lag2','breath_id_lagsame',\n            'breath_id_lag2same'], axis=1, inplace=True)\n\ntest = test.drop(['id', 'breath_id','one','count','breath_id_lag',\n                  'breath_id_lag2','breath_id_lagsame',\n                  'breath_id_lag2same'], axis=1)","metadata":{"id":"431a7ee7","execution":{"iopub.status.busy":"2021-10-14T04:33:42.674772Z","iopub.execute_input":"2021-10-14T04:33:42.675372Z","iopub.status.idle":"2021-10-14T04:33:46.768422Z","shell.execute_reply.started":"2021-10-14T04:33:42.675317Z","shell.execute_reply":"2021-10-14T04:33:46.767663Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"RS = RobustScaler()\ntrain = RS.fit_transform(train)\ntest = RS.transform(test)","metadata":{"id":"d24ef65c","execution":{"iopub.status.busy":"2021-10-14T04:33:46.769844Z","iopub.execute_input":"2021-10-14T04:33:46.770183Z","iopub.status.idle":"2021-10-14T04:33:58.460545Z","shell.execute_reply.started":"2021-10-14T04:33:46.770141Z","shell.execute_reply":"2021-10-14T04:33:58.459483Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"code","source":"train = train.reshape(-1, 80, train.shape[-1])\ntest = test.reshape(-1, 80, train.shape[-1])","metadata":{"id":"6f1a3e19","execution":{"iopub.status.busy":"2021-10-14T04:33:58.461677Z","iopub.execute_input":"2021-10-14T04:33:58.462694Z","iopub.status.idle":"2021-10-14T04:33:58.467934Z","shell.execute_reply.started":"2021-10-14T04:33:58.462656Z","shell.execute_reply":"2021-10-14T04:33:58.466874Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras import layers\nfrom tensorflow import keras","metadata":{"id":"m4lOIi9HdExc","execution":{"iopub.status.busy":"2021-10-14T04:33:58.470894Z","iopub.execute_input":"2021-10-14T04:33:58.471121Z","iopub.status.idle":"2021-10-14T04:33:58.481269Z","shell.execute_reply.started":"2021-10-14T04:33:58.471094Z","shell.execute_reply":"2021-10-14T04:33:58.480214Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"# refer to https://www.tensorflow.org/text/tutorials/transformer#encoder\ndef get_angles(pos, i, d_model):\n  angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))\n  return pos * angle_rates\n\ndef get_positional_encoding(position, d_model):\n  angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n                          np.arange(d_model)[np.newaxis, :],\n                          d_model)\n\n  # apply sin to even indices in the array; 2i\n  angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n\n  # apply cos to odd indices in the array; 2i+1\n  angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n\n  pos_encoding = angle_rads[np.newaxis, ...]\n\n  return tf.cast(pos_encoding, dtype=tf.float32)","metadata":{"id":"dqA2324UJKb_","execution":{"iopub.status.busy":"2021-10-14T04:33:58.482811Z","iopub.execute_input":"2021-10-14T04:33:58.483087Z","iopub.status.idle":"2021-10-14T04:33:58.493507Z","shell.execute_reply.started":"2021-10-14T04:33:58.483054Z","shell.execute_reply":"2021-10-14T04:33:58.492738Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"code","source":"#refer to https://keras.io/examples/timeseries/timeseries_classification_transformer/\ndef transformer_encoder(inputs, key_dim, num_heads, ff_dim, dropout=0):\n    # Normalization and Attention\n    x = layers.LayerNormalization(epsilon=1e-6)(inputs)\n        \n    x = layers.MultiHeadAttention(\n        key_dim=key_dim, num_heads=num_heads, dropout=dropout\n    )(x, x)\n    x = layers.Dropout(dropout)(x)\n    res = x + inputs\n\n    # Feed Forward Part\n    # Feed forward can be Den lay or RNN as long as it keep final dimension\n    x = layers.LayerNormalization(epsilon=1e-6)(res)\n    x = layers.Conv1D(filters=ff_dim, kernel_size=1, activation=\"selu\")(x)\n    x = layers.Dropout(dropout)(x)\n    x = layers.Conv1D(filters=inputs.shape[-1], kernel_size=1)(x)\n    return x + res","metadata":{"id":"i5MeMc-0ddTj","execution":{"iopub.status.busy":"2021-10-14T04:33:58.494811Z","iopub.execute_input":"2021-10-14T04:33:58.495085Z","iopub.status.idle":"2021-10-14T04:33:58.506213Z","shell.execute_reply.started":"2021-10-14T04:33:58.495055Z","shell.execute_reply":"2021-10-14T04:33:58.505501Z"},"trusted":true},"execution_count":16,"outputs":[]},{"cell_type":"code","source":"# refer to https://www.tensorflow.org/text/tutorials/transformer#encoder\n# https://rubikscode.net/2019/08/19/transformer-with-python-and-tensorflow-2-0-encoder-decoder/\n# https://trungtran.io/2019/04/29/create-the-transformer-with-tensorflow-2-0/\n# https://github.com/ChunML/NLP/blob/master/chatbot/model.py\ndef build_transfer_model(\n    input_shape,\n    key_dims=[512, 256, 128, 64],\n    num_heads=[64, 64, 64, 64],\n    ff_dims=[512, 256, 128, 64],\n    mlp_units=[32, 1],\n    dropout=0,\n    mlp_dropout=0,\n):\n\n    inputs = keras.Input(shape=input_shape)\n    time_lenght = input_shape[0]\n    positional_encoding = get_positional_encoding(time_lenght, input_shape[-1])\n#     print('positional_encoding:', positional_encoding)\n\n    x = inputs\n\n    # increase x then add position encoding.\n    '''\n    The reason we increase the embedding values before the addition is to make \n    the positional encoding relatively smaller. This means the original meaning \n    in the embedding vector won’t be lost when we add them together.\n    '''\n    x *= tf.math.sqrt(tf.cast(input_shape[-1], tf.float32))\n    x += positional_encoding[:, :time_lenght, :]\n\n    for key_dim, ff_dim, num_head in zip(key_dims, ff_dims, num_heads):\n        x = transformer_encoder(x, key_dim, num_head, ff_dim, dropout)\n\n    x_pool = layers.GlobalAveragePooling1D(data_format=\"channels_first\")(x)\n    x_pool = tf.expand_dims(x_pool, -1)\n\n    for dim in mlp_units:\n        x = layers.Dense(dim, activation=\"selu\")(x)\n        x = layers.Dropout(mlp_dropout)(x)\n\n    outputs = layers.Dense(mlp_units[-1])(x + x_pool)\n    model = keras.Model(inputs, outputs)\n    model.summary()\n    return model","metadata":{"id":"b1uCDdywdhRH","execution":{"iopub.status.busy":"2021-10-14T04:33:58.507775Z","iopub.execute_input":"2021-10-14T04:33:58.508091Z","iopub.status.idle":"2021-10-14T04:33:58.522763Z","shell.execute_reply.started":"2021-10-14T04:33:58.508052Z","shell.execute_reply":"2021-10-14T04:33:58.521659Z"},"trusted":true},"execution_count":17,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.utils import plot_model\nmodel = build_transfer_model(train.shape[1:])\nplot_model(\n    model, \n    to_file='Google_Brain_Keras_Model.png', \n    show_shapes=True,\n    show_layer_names=True\n)","metadata":{"id":"BvagsDTSiXYs","outputId":"3fe8de35-7981-4d7b-f5b2-1a6152cd5a50","execution":{"iopub.status.busy":"2021-10-14T04:33:58.524144Z","iopub.execute_input":"2021-10-14T04:33:58.524466Z","iopub.status.idle":"2021-10-14T04:34:00.396138Z","shell.execute_reply.started":"2021-10-14T04:33:58.524438Z","shell.execute_reply":"2021-10-14T04:34:00.394993Z"},"trusted":true},"execution_count":18,"outputs":[]},{"cell_type":"code","source":"# Function to get hardware strategy\ndef get_hardware_strategy():\n    try:\n        # TPU detection. No parameters necessary if TPU_NAME environment variable is\n        # set: this is always the case on Kaggle.\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        tpu = None\n\n    if tpu:\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        tf.config.optimizer.set_jit(True)\n    else:\n        # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n        strategy = tf.distribute.get_strategy()\n\n    return tpu, strategy\n\ntpu, strategy = get_hardware_strategy()","metadata":{"id":"2WeSVrUJUWs-","execution":{"iopub.status.busy":"2021-10-14T04:34:00.397842Z","iopub.execute_input":"2021-10-14T04:34:00.398104Z","iopub.status.idle":"2021-10-14T04:34:05.844724Z","shell.execute_reply.started":"2021-10-14T04:34:00.398073Z","shell.execute_reply":"2021-10-14T04:34:05.844056Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.optimizers.schedules import ExponentialDecay\nEPOCH = 300\nBATCH_SIZE = 512\nNUM_FOLDS = 5\n\nwith strategy.scope():\n    kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=2021)\n    test_preds = []\n    train_preds = []\n    for fold, (train_idx, test_idx) in enumerate(kf.split(train, targets)):\n        print('-'*15, '>', f'Fold {fold+1}', '<', '-'*15)\n        X_train, X_valid = train[train_idx], train[test_idx]\n        y_train, y_valid = targets[train_idx], targets[test_idx]\n        #u_out_train, u_out_valid = u_outs[train_idx], u_outs[test_idx]\n        \n        #model = get_cnn_model(train)\n        model = build_transfer_model(train.shape[1:])\n        model.compile(optimizer='adam', \n                      # loss=GBVPP_loss,\n                      loss=\"mae\"\n                      )\n\n        #scheduler = ExponentialDecay(1e-3, 40*((len(train)*0.8)/BATCH_SIZE), 1e-5)\n        #lr = LearningRateScheduler(scheduler, verbose=1)\n        lr = ReduceLROnPlateau(monitor=\"val_loss\", factor=0.75, patience=10, verbose=1)\n        es = EarlyStopping(monitor=\"val_loss\", patience=50, verbose=1, mode=\"min\", restore_best_weights=True)\n        checkpoint_filepath = f\"folds{fold}.hdf5\"\n        sv = keras.callbacks.ModelCheckpoint(\n            checkpoint_filepath, monitor='val_loss', verbose=1, save_best_only=True,\n            save_weights_only=False, mode='auto', save_freq='epoch',\n            options=None)\n\n        history = model.fit(X_train,\n                            y_train,\n                            # validation_data=(X_valid, np.append(y_valid, u_out_valid, axis =1)), \n                            validation_data=(X_valid, y_valid), \n                            epochs=EPOCH, \n                            batch_size=1024, \n                            callbacks=[lr,\n                                       es,\n                                       sv])\n        \n        if IN_COLAB:\n          with open(model_folder+checkpoint_filepath, 'wb') as f:\n            f.write(open(checkpoint_filepath, 'rb').read())\n    \n        test_preds.append(model.predict(test).squeeze().reshape(-1, 1).squeeze())\n        train_preds.append(model.predict(train).squeeze().reshape(-1, 1).squeeze())\n","metadata":{"id":"12a089a5","outputId":"a398c6a2-9197-4830-e2a8-d08d2e3b917e","execution":{"iopub.status.busy":"2021-10-14T04:34:05.846035Z","iopub.execute_input":"2021-10-14T04:34:05.846375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mae(targets, np.median(np.vstack(test_preds),axis=0))","metadata":{"id":"0ZR_KtDCXk-c","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PRESSURE_STEP = 0.07030214545120961\nPRESSURE_MIN = -1.8957442945646408\nPRESSURE_MAX = 64.82099173863948","metadata":{"id":"7025c5f6","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ENSEMBLE FOLDS WITH MEAN\nsubmission[\"pressure\"] = sum(test_preds)/NUM_FOLDS\nsubmission.to_csv('submission_mean.csv', index=False)","metadata":{"id":"41dffad5","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ENSEMBLE FOLDS WITH MEDIAN\nsubmission[\"pressure\"] = np.median(np.vstack(test_preds),axis=0)\nsubmission.to_csv('submission_median.csv', index=False)","metadata":{"id":"6bb56003","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ENSEMBLE FOLDS WITH MEDIAN AND ROUND PREDICTIONS\nsubmission[\"pressure\"] =\\\n    np.round( (submission.pressure - PRESSURE_MIN)/PRESSURE_STEP ) * PRESSURE_STEP + PRESSURE_MIN\nsubmission.pressure = np.clip(submission.pressure, PRESSURE_MIN, PRESSURE_MAX)\nsubmission.to_csv('submission_median_round.csv', index=False)","metadata":{"id":"9236b88e","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if IN_COLAB:\n  !zip cnn_models.zip *.hdf5\n  !zip submission.zip *.csv\n  with open(model_folder + 'submission.zip', 'wb') as f:\n    f.write(open('submission.zip', 'rb').read())\n  with open(model_folder + 'cnn_models.zip', 'wb') as f:\n    f.write(open('cnn_models.zip', 'rb').read())","metadata":{"id":"K5z9iaJ--P8P","trusted":true},"execution_count":null,"outputs":[]}]}