sparse_features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import sklearn\n",
    "import math"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Table: (232744, 19)\n",
      "Columns: Index(['Row', 'Anon Student Id', 'Problem Hierarchy', 'Problem Name',\n",
      "       'Problem View', 'Step Name', 'Step Start Time',\n",
      "       'First Transaction Time', 'Correct Transaction Time', 'Step End Time',\n",
      "       'Step Duration (sec)', 'Correct Step Duration (sec)',\n",
      "       'Error Step Duration (sec)', 'Correct First Attempt', 'Incorrects',\n",
      "       'Hints', 'Corrects', 'KC(Default)', 'Opportunity(Default)'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "# Load training data\n",
    "train_filepath = \"data/train.csv\"\n",
    "traindf = pd.read_csv(train_filepath, sep='\\t')\n",
    "print(\"Table:\", np.shape(traindf))\n",
    "print(\"Columns:\", traindf.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Table: (1140, 19)\n",
      "Columns: Index(['Row', 'Anon Student Id', 'Problem Hierarchy', 'Problem Name',\n",
      "       'Problem View', 'Step Name', 'Step Start Time',\n",
      "       'First Transaction Time', 'Correct Transaction Time', 'Step End Time',\n",
      "       'Step Duration (sec)', 'Correct Step Duration (sec)',\n",
      "       'Error Step Duration (sec)', 'Correct First Attempt', 'Incorrects',\n",
      "       'Hints', 'Corrects', 'KC(Default)', 'Opportunity(Default)'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "# Load testing data\n",
    "test_filepath = \"data/test.csv\"\n",
    "testdf = pd.read_csv(test_filepath, sep='\\t')\n",
    "print(\"Table:\", np.shape(testdf))\n",
    "print(\"Columns:\", testdf.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Concatenate training set and testing set\n",
    "# First combine two set to ensure the one-hot keys form a universal set of all categories\n",
    "df = pd.concat((traindf, testdf), axis=0, sort=False)\n",
    "\n",
    "# Cast columns\n",
    "df[\"KC(Default)\"] = df[\"KC(Default)\"].astype(str)\n",
    "df[\"Opportunity(Default)\"] = df[\"Opportunity(Default)\"].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Table: (233884, 19)\n"
     ]
    }
   ],
   "source": [
    "print(\"Table:\", df.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split unit and section out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split by comma\n",
    "df.insert(3, \"Problem Unit\", df.apply(lambda row: row[\"Problem Hierarchy\"].split(',')[0].strip(), axis=1))\n",
    "df.insert(4, \"Problem Section\", df.apply(lambda row: row[\"Problem Hierarchy\"].split(',')[1].strip(), axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Table: (233884, 21)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Row</th>\n",
       "      <th>Anon Student Id</th>\n",
       "      <th>Problem Hierarchy</th>\n",
       "      <th>Problem Unit</th>\n",
       "      <th>Problem Section</th>\n",
       "      <th>Problem Name</th>\n",
       "      <th>Problem View</th>\n",
       "      <th>Step Name</th>\n",
       "      <th>Step Start Time</th>\n",
       "      <th>First Transaction Time</th>\n",
       "      <th>...</th>\n",
       "      <th>Step End Time</th>\n",
       "      <th>Step Duration (sec)</th>\n",
       "      <th>Correct Step Duration (sec)</th>\n",
       "      <th>Error Step Duration (sec)</th>\n",
       "      <th>Correct First Attempt</th>\n",
       "      <th>Incorrects</th>\n",
       "      <th>Hints</th>\n",
       "      <th>Corrects</th>\n",
       "      <th>KC(Default)</th>\n",
       "      <th>Opportunity(Default)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9938</td>\n",
       "      <td>52vEY7f17k</td>\n",
       "      <td>Unit CTA1_13, Section CTA1_13-1</td>\n",
       "      <td>Unit CTA1_13</td>\n",
       "      <td>Section CTA1_13-1</td>\n",
       "      <td>PROP04</td>\n",
       "      <td>1</td>\n",
       "      <td>R1C1</td>\n",
       "      <td>2005-09-09 12:23:34.0</td>\n",
       "      <td>2005-09-09 12:24:07.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2005-09-09 12:24:07.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9939</td>\n",
       "      <td>52vEY7f17k</td>\n",
       "      <td>Unit CTA1_13, Section CTA1_13-1</td>\n",
       "      <td>Unit CTA1_13</td>\n",
       "      <td>Section CTA1_13-1</td>\n",
       "      <td>PROP04</td>\n",
       "      <td>1</td>\n",
       "      <td>R1C2</td>\n",
       "      <td>2005-09-09 12:24:07.0</td>\n",
       "      <td>2005-09-09 12:24:22.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2005-09-09 12:24:22.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9940</td>\n",
       "      <td>52vEY7f17k</td>\n",
       "      <td>Unit CTA1_13, Section CTA1_13-1</td>\n",
       "      <td>Unit CTA1_13</td>\n",
       "      <td>Section CTA1_13-1</td>\n",
       "      <td>PROP04</td>\n",
       "      <td>1</td>\n",
       "      <td>R3C1</td>\n",
       "      <td>2005-09-09 12:24:22.0</td>\n",
       "      <td>2005-09-09 12:25:16.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2005-09-09 12:25:40.0</td>\n",
       "      <td>78.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>78.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Define Variable</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9941</td>\n",
       "      <td>52vEY7f17k</td>\n",
       "      <td>Unit CTA1_13, Section CTA1_13-1</td>\n",
       "      <td>Unit CTA1_13</td>\n",
       "      <td>Section CTA1_13-1</td>\n",
       "      <td>PROP04</td>\n",
       "      <td>1</td>\n",
       "      <td>R3C2</td>\n",
       "      <td>2005-09-09 12:25:40.0</td>\n",
       "      <td>2005-09-09 12:25:51.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2005-09-09 12:27:24.0</td>\n",
       "      <td>104.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>104.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Using small numbers~~Write expression, positiv...</td>\n",
       "      <td>1~~1~~1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9942</td>\n",
       "      <td>52vEY7f17k</td>\n",
       "      <td>Unit CTA1_13, Section CTA1_13-1</td>\n",
       "      <td>Unit CTA1_13</td>\n",
       "      <td>Section CTA1_13-1</td>\n",
       "      <td>PROP04</td>\n",
       "      <td>1</td>\n",
       "      <td>R4C1</td>\n",
       "      <td>2005-09-09 12:27:24.0</td>\n",
       "      <td>2005-09-09 12:27:30.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2005-09-09 12:27:30.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Entering a given</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Row Anon Student Id                Problem Hierarchy  Problem Unit  \\\n",
       "0  9938      52vEY7f17k  Unit CTA1_13, Section CTA1_13-1  Unit CTA1_13   \n",
       "1  9939      52vEY7f17k  Unit CTA1_13, Section CTA1_13-1  Unit CTA1_13   \n",
       "2  9940      52vEY7f17k  Unit CTA1_13, Section CTA1_13-1  Unit CTA1_13   \n",
       "3  9941      52vEY7f17k  Unit CTA1_13, Section CTA1_13-1  Unit CTA1_13   \n",
       "4  9942      52vEY7f17k  Unit CTA1_13, Section CTA1_13-1  Unit CTA1_13   \n",
       "\n",
       "     Problem Section Problem Name  Problem View Step Name  \\\n",
       "0  Section CTA1_13-1       PROP04             1      R1C1   \n",
       "1  Section CTA1_13-1       PROP04             1      R1C2   \n",
       "2  Section CTA1_13-1       PROP04             1      R3C1   \n",
       "3  Section CTA1_13-1       PROP04             1      R3C2   \n",
       "4  Section CTA1_13-1       PROP04             1      R4C1   \n",
       "\n",
       "         Step Start Time First Transaction Time  ...          Step End Time  \\\n",
       "0  2005-09-09 12:23:34.0  2005-09-09 12:24:07.0  ...  2005-09-09 12:24:07.0   \n",
       "1  2005-09-09 12:24:07.0  2005-09-09 12:24:22.0  ...  2005-09-09 12:24:22.0   \n",
       "2  2005-09-09 12:24:22.0  2005-09-09 12:25:16.0  ...  2005-09-09 12:25:40.0   \n",
       "3  2005-09-09 12:25:40.0  2005-09-09 12:25:51.0  ...  2005-09-09 12:27:24.0   \n",
       "4  2005-09-09 12:27:24.0  2005-09-09 12:27:30.0  ...  2005-09-09 12:27:30.0   \n",
       "\n",
       "  Step Duration (sec)  Correct Step Duration (sec)  Error Step Duration (sec)  \\\n",
       "0                33.0                         33.0                        NaN   \n",
       "1                15.0                         15.0                        NaN   \n",
       "2                78.0                          NaN                       78.0   \n",
       "3               104.0                          NaN                      104.0   \n",
       "4                 6.0                          6.0                        NaN   \n",
       "\n",
       "   Correct First Attempt  Incorrects  Hints  Corrects  \\\n",
       "0                    1.0         0.0    0.0       1.0   \n",
       "1                    1.0         0.0    0.0       1.0   \n",
       "2                    0.0         2.0    0.0       1.0   \n",
       "3                    0.0         4.0    9.0       1.0   \n",
       "4                    1.0         0.0    0.0       1.0   \n",
       "\n",
       "                                         KC(Default) Opportunity(Default)  \n",
       "0                                                nan                  nan  \n",
       "1                                                nan                  nan  \n",
       "2                                    Define Variable                    1  \n",
       "3  Using small numbers~~Write expression, positiv...              1~~1~~1  \n",
       "4                                   Entering a given                    1  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Overview of train data\n",
    "print(\"Table:\", df.shape)\n",
    "df.head()\n",
    "# df[[\"Problem Name\", \"Step Name\", \"Incorrects\", \"Hints\", \"Corrects\"]].head(50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## One-hot feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_dim_split___(df, col):\n",
    "    cols = set(df[col])\n",
    "    numcols = len(cols)\n",
    "    print(\"Column %s has %d values, so we will be adding as many columns to this dataframe\" % (col, numcols))\n",
    "    \n",
    "    # Row number\n",
    "    numrows = len(df)\n",
    "\n",
    "    # Create df\n",
    "    newdf = pd.DataFrame(np.zeros((numrows, numcols)), index=df.index, columns=list(cols))\n",
    "\n",
    "    # For each value in the dataframe, mark the corresponding column as 1\n",
    "    for c in cols:\n",
    "        newdf.loc[df[col] == c, c] = 1\n",
    "\n",
    "    return newdf\n",
    "\n",
    "def one_dim_split(df, col, cata=None):\n",
    "    dummies = pd.get_dummies(df[col], prefix=cata, sparse=True)\n",
    "    print(\"Column %s has %d values, so we will add as many columns to this dataframe.\" % (col, dummies.shape[1]))\n",
    "    return dummies"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### One-dimensional Feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column Problem Unit has 32 values, so we will add as many columns to this dataframe.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unit CTA1_01</th>\n",
       "      <th>Unit CTA1_02</th>\n",
       "      <th>Unit CTA1_04</th>\n",
       "      <th>Unit CTA1_06</th>\n",
       "      <th>Unit CTA1_08</th>\n",
       "      <th>Unit CTA1_10</th>\n",
       "      <th>Unit CTA1_12</th>\n",
       "      <th>Unit CTA1_13</th>\n",
       "      <th>Unit CTA1_14</th>\n",
       "      <th>Unit CTA1_15</th>\n",
       "      <th>...</th>\n",
       "      <th>Unit ES_06</th>\n",
       "      <th>Unit ES_07</th>\n",
       "      <th>Unit EXPT-PRODUCT-SIMP-A_ES</th>\n",
       "      <th>Unit EXPT-QUOTIENT-SIMP-A_ES</th>\n",
       "      <th>Unit LINEAR-INEQUALITY-GRAPHING</th>\n",
       "      <th>Unit QUAD-ADD-AREA-ALG1</th>\n",
       "      <th>Unit QUAD-VERTICAL-MOTION</th>\n",
       "      <th>Unit QUADRATICS-FACTORING_ES</th>\n",
       "      <th>Unit QUADRATICS-SOLVING_ES</th>\n",
       "      <th>Unit QuadLinearFuncTrans</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unit CTA1_01  Unit CTA1_02  Unit CTA1_04  Unit CTA1_06  Unit CTA1_08  \\\n",
       "0             0             0             0             0             0   \n",
       "1             0             0             0             0             0   \n",
       "2             0             0             0             0             0   \n",
       "3             0             0             0             0             0   \n",
       "4             0             0             0             0             0   \n",
       "\n",
       "   Unit CTA1_10  Unit CTA1_12  Unit CTA1_13  Unit CTA1_14  Unit CTA1_15  ...  \\\n",
       "0             0             0             1             0             0  ...   \n",
       "1             0             0             1             0             0  ...   \n",
       "2             0             0             1             0             0  ...   \n",
       "3             0             0             1             0             0  ...   \n",
       "4             0             0             1             0             0  ...   \n",
       "\n",
       "   Unit ES_06  Unit ES_07  Unit EXPT-PRODUCT-SIMP-A_ES  \\\n",
       "0           0           0                            0   \n",
       "1           0           0                            0   \n",
       "2           0           0                            0   \n",
       "3           0           0                            0   \n",
       "4           0           0                            0   \n",
       "\n",
       "   Unit EXPT-QUOTIENT-SIMP-A_ES  Unit LINEAR-INEQUALITY-GRAPHING  \\\n",
       "0                             0                                0   \n",
       "1                             0                                0   \n",
       "2                             0                                0   \n",
       "3                             0                                0   \n",
       "4                             0                                0   \n",
       "\n",
       "   Unit QUAD-ADD-AREA-ALG1  Unit QUAD-VERTICAL-MOTION  \\\n",
       "0                        0                          0   \n",
       "1                        0                          0   \n",
       "2                        0                          0   \n",
       "3                        0                          0   \n",
       "4                        0                          0   \n",
       "\n",
       "   Unit QUADRATICS-FACTORING_ES  Unit QUADRATICS-SOLVING_ES  \\\n",
       "0                             0                           0   \n",
       "1                             0                           0   \n",
       "2                             0                           0   \n",
       "3                             0                           0   \n",
       "4                             0                           0   \n",
       "\n",
       "   Unit QuadLinearFuncTrans  \n",
       "0                         0  \n",
       "1                         0  \n",
       "2                         0  \n",
       "3                         0  \n",
       "4                         0  \n",
       "\n",
       "[5 rows x 32 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Problem Unit\n",
    "pu_features = one_dim_split(df, \"Problem Unit\")\n",
    "pu_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column Problem Section has 138 values, so we will add as many columns to this dataframe.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Section CTA1_01-1</th>\n",
       "      <th>Section CTA1_01-2</th>\n",
       "      <th>Section CTA1_01-3</th>\n",
       "      <th>Section CTA1_01-4</th>\n",
       "      <th>Section CTA1_02-1</th>\n",
       "      <th>Section CTA1_02-2</th>\n",
       "      <th>Section CTA1_02-3</th>\n",
       "      <th>Section CTA1_02-4</th>\n",
       "      <th>Section CTA1_04-1</th>\n",
       "      <th>Section CTA1_04-2</th>\n",
       "      <th>...</th>\n",
       "      <th>Section QLFuncTrans5</th>\n",
       "      <th>Section QLFuncTrans6</th>\n",
       "      <th>Section QLFuncTrans7</th>\n",
       "      <th>Section QLFuncTrans8</th>\n",
       "      <th>Section QLFuncTrans9</th>\n",
       "      <th>Section QUAD-ADD-AREA-ALG1-1</th>\n",
       "      <th>Section QUAD-VERTICAL-MOTION-1</th>\n",
       "      <th>Section QUADRATICS-FACTORING_ES-1</th>\n",
       "      <th>Section QUADRATICS-FACTORING_ES-2</th>\n",
       "      <th>Section QUADRATICS-SOLVING_ES-1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 138 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Section CTA1_01-1  Section CTA1_01-2  Section CTA1_01-3  Section CTA1_01-4  \\\n",
       "0                  0                  0                  0                  0   \n",
       "1                  0                  0                  0                  0   \n",
       "2                  0                  0                  0                  0   \n",
       "3                  0                  0                  0                  0   \n",
       "4                  0                  0                  0                  0   \n",
       "\n",
       "   Section CTA1_02-1  Section CTA1_02-2  Section CTA1_02-3  Section CTA1_02-4  \\\n",
       "0                  0                  0                  0                  0   \n",
       "1                  0                  0                  0                  0   \n",
       "2                  0                  0                  0                  0   \n",
       "3                  0                  0                  0                  0   \n",
       "4                  0                  0                  0                  0   \n",
       "\n",
       "   Section CTA1_04-1  Section CTA1_04-2  ...  Section QLFuncTrans5  \\\n",
       "0                  0                  0  ...                     0   \n",
       "1                  0                  0  ...                     0   \n",
       "2                  0                  0  ...                     0   \n",
       "3                  0                  0  ...                     0   \n",
       "4                  0                  0  ...                     0   \n",
       "\n",
       "   Section QLFuncTrans6  Section QLFuncTrans7  Section QLFuncTrans8  \\\n",
       "0                     0                     0                     0   \n",
       "1                     0                     0                     0   \n",
       "2                     0                     0                     0   \n",
       "3                     0                     0                     0   \n",
       "4                     0                     0                     0   \n",
       "\n",
       "   Section QLFuncTrans9  Section QUAD-ADD-AREA-ALG1-1  \\\n",
       "0                     0                             0   \n",
       "1                     0                             0   \n",
       "2                     0                             0   \n",
       "3                     0                             0   \n",
       "4                     0                             0   \n",
       "\n",
       "   Section QUAD-VERTICAL-MOTION-1  Section QUADRATICS-FACTORING_ES-1  \\\n",
       "0                               0                                  0   \n",
       "1                               0                                  0   \n",
       "2                               0                                  0   \n",
       "3                               0                                  0   \n",
       "4                               0                                  0   \n",
       "\n",
       "   Section QUADRATICS-FACTORING_ES-2  Section QUADRATICS-SOLVING_ES-1  \n",
       "0                                  0                                0  \n",
       "1                                  0                                0  \n",
       "2                                  0                                0  \n",
       "3                                  0                                0  \n",
       "4                                  0                                0  \n",
       "\n",
       "[5 rows x 138 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Problem Section\n",
    "ps_features = one_dim_split(df, \"Problem Section\")\n",
    "ps_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column Problem Name has 1023 values, so we will add as many columns to this dataframe.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1PTB02</th>\n",
       "      <th>1PTB03</th>\n",
       "      <th>1PTB04</th>\n",
       "      <th>1PTB05</th>\n",
       "      <th>1PTB06</th>\n",
       "      <th>1PTB07</th>\n",
       "      <th>1PTB08</th>\n",
       "      <th>1PTB09</th>\n",
       "      <th>1PTFB10</th>\n",
       "      <th>1PTFB11</th>\n",
       "      <th>...</th>\n",
       "      <th>YGT-4X-10</th>\n",
       "      <th>YGT5X+12</th>\n",
       "      <th>YGTX-7</th>\n",
       "      <th>YLE-3X-13</th>\n",
       "      <th>YLE-7X+20</th>\n",
       "      <th>YLEX+7</th>\n",
       "      <th>YLT-X-4</th>\n",
       "      <th>YLT250</th>\n",
       "      <th>YLT2X-5</th>\n",
       "      <th>YLT3X+5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1023 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   1PTB02  1PTB03  1PTB04  1PTB05  1PTB06  1PTB07  1PTB08  1PTB09  1PTFB10  \\\n",
       "0       0       0       0       0       0       0       0       0        0   \n",
       "1       0       0       0       0       0       0       0       0        0   \n",
       "2       0       0       0       0       0       0       0       0        0   \n",
       "3       0       0       0       0       0       0       0       0        0   \n",
       "4       0       0       0       0       0       0       0       0        0   \n",
       "\n",
       "   1PTFB11  ...  YGT-4X-10  YGT5X+12  YGTX-7  YLE-3X-13  YLE-7X+20  YLEX+7  \\\n",
       "0        0  ...          0         0       0          0          0       0   \n",
       "1        0  ...          0         0       0          0          0       0   \n",
       "2        0  ...          0         0       0          0          0       0   \n",
       "3        0  ...          0         0       0          0          0       0   \n",
       "4        0  ...          0         0       0          0          0       0   \n",
       "\n",
       "   YLT-X-4  YLT250  YLT2X-5  YLT3X+5  \n",
       "0        0       0        0        0  \n",
       "1        0       0        0        0  \n",
       "2        0       0        0        0  \n",
       "3        0       0        0        0  \n",
       "4        0       0        0        0  \n",
       "\n",
       "[5 rows x 1023 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Problem Name\n",
    "pn_features = one_dim_split(df, \"Problem Name\")\n",
    "pn_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column Step Name has 60934 values, so we will add as many columns to this dataframe.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>((2y-3)/2)/2 = ((10/2)/2)/2</th>\n",
       "      <th>((2y-3)/2)/2 = (5/2)/2</th>\n",
       "      <th>((7r+y)/7)/(1/7) = (b/7)/(1/7)</th>\n",
       "      <th>((m*h+56)/m)/(1/m) = (q/m)/(1/m)</th>\n",
       "      <th>((s*v+r)/s)/(1/s) = (y/s)/(1/s)</th>\n",
       "      <th>((y-3)-m*-7)/m = m*x/m</th>\n",
       "      <th>(-(-13y+1)/13)/(-1/13) = (1/13)/(-1/13)</th>\n",
       "      <th>(-(-5)+sqrt(5^2-4*1*6))/2 = H</th>\n",
       "      <th>(-(-5)+sqrt(5^2-4*3*1))/6 = x</th>\n",
       "      <th>(-(-5)-sqrt(5^2-12))/6 = x</th>\n",
       "      <th>...</th>\n",
       "      <th>z-y = c</th>\n",
       "      <th>z-y = c+y-y</th>\n",
       "      <th>z-y = y+c-y</th>\n",
       "      <th>z-z-c = y-z</th>\n",
       "      <th>z/(-u) = -u*y/(-u)</th>\n",
       "      <th>z/y = -u</th>\n",
       "      <th>z/y*y = -u*y</th>\n",
       "      <th>z1R1</th>\n",
       "      <th>z2R1</th>\n",
       "      <th>zR2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 60934 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   ((2y-3)/2)/2 = ((10/2)/2)/2  ((2y-3)/2)/2 = (5/2)/2  \\\n",
       "0                            0                       0   \n",
       "1                            0                       0   \n",
       "2                            0                       0   \n",
       "3                            0                       0   \n",
       "4                            0                       0   \n",
       "\n",
       "   ((7r+y)/7)/(1/7) = (b/7)/(1/7)  ((m*h+56)/m)/(1/m) = (q/m)/(1/m)  \\\n",
       "0                               0                                 0   \n",
       "1                               0                                 0   \n",
       "2                               0                                 0   \n",
       "3                               0                                 0   \n",
       "4                               0                                 0   \n",
       "\n",
       "   ((s*v+r)/s)/(1/s) = (y/s)/(1/s)  ((y-3)-m*-7)/m = m*x/m  \\\n",
       "0                                0                       0   \n",
       "1                                0                       0   \n",
       "2                                0                       0   \n",
       "3                                0                       0   \n",
       "4                                0                       0   \n",
       "\n",
       "   (-(-13y+1)/13)/(-1/13) = (1/13)/(-1/13)  (-(-5)+sqrt(5^2-4*1*6))/2 = H  \\\n",
       "0                                        0                              0   \n",
       "1                                        0                              0   \n",
       "2                                        0                              0   \n",
       "3                                        0                              0   \n",
       "4                                        0                              0   \n",
       "\n",
       "   (-(-5)+sqrt(5^2-4*3*1))/6 = x  (-(-5)-sqrt(5^2-12))/6 = x  ...  z-y = c  \\\n",
       "0                              0                           0  ...        0   \n",
       "1                              0                           0  ...        0   \n",
       "2                              0                           0  ...        0   \n",
       "3                              0                           0  ...        0   \n",
       "4                              0                           0  ...        0   \n",
       "\n",
       "   z-y = c+y-y  z-y = y+c-y  z-z-c = y-z  z/(-u) = -u*y/(-u)  z/y = -u  \\\n",
       "0            0            0            0                   0         0   \n",
       "1            0            0            0                   0         0   \n",
       "2            0            0            0                   0         0   \n",
       "3            0            0            0                   0         0   \n",
       "4            0            0            0                   0         0   \n",
       "\n",
       "   z/y*y = -u*y  z1R1  z2R1  zR2  \n",
       "0             0     0     0    0  \n",
       "1             0     0     0    0  \n",
       "2             0     0     0    0  \n",
       "3             0     0     0    0  \n",
       "4             0     0     0    0  \n",
       "\n",
       "[5 rows x 60934 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Step Name\n",
    "sn_features = one_dim_split(df, \"Step Name\")\n",
    "sn_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column Anon Student Id has 174 values, so we will add as many columns to this dataframe.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>02i5jCrfQK</th>\n",
       "      <th>0GlR30c2Mt</th>\n",
       "      <th>0KS4yy9G96</th>\n",
       "      <th>0nF0z1401O</th>\n",
       "      <th>12M70dm49w</th>\n",
       "      <th>162lfGskK2</th>\n",
       "      <th>16zN4n2fFO</th>\n",
       "      <th>183ppvOXp8</th>\n",
       "      <th>1bJbgQ32E3</th>\n",
       "      <th>1k3GIfwBw6</th>\n",
       "      <th>...</th>\n",
       "      <th>xH487WMLoS</th>\n",
       "      <th>y5009rpZ12</th>\n",
       "      <th>yA79w15tnU</th>\n",
       "      <th>yB480DNZ70</th>\n",
       "      <th>yDO9pl8GC0</th>\n",
       "      <th>yG447121nm</th>\n",
       "      <th>z2zuhnARi6</th>\n",
       "      <th>z9svx3mA4s</th>\n",
       "      <th>zUlk9c7UEv</th>\n",
       "      <th>zfYl0YECr0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 174 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   02i5jCrfQK  0GlR30c2Mt  0KS4yy9G96  0nF0z1401O  12M70dm49w  162lfGskK2  \\\n",
       "0           0           0           0           0           0           0   \n",
       "1           0           0           0           0           0           0   \n",
       "2           0           0           0           0           0           0   \n",
       "3           0           0           0           0           0           0   \n",
       "4           0           0           0           0           0           0   \n",
       "\n",
       "   16zN4n2fFO  183ppvOXp8  1bJbgQ32E3  1k3GIfwBw6  ...  xH487WMLoS  \\\n",
       "0           0           0           0           0  ...           0   \n",
       "1           0           0           0           0  ...           0   \n",
       "2           0           0           0           0  ...           0   \n",
       "3           0           0           0           0  ...           0   \n",
       "4           0           0           0           0  ...           0   \n",
       "\n",
       "   y5009rpZ12  yA79w15tnU  yB480DNZ70  yDO9pl8GC0  yG447121nm  z2zuhnARi6  \\\n",
       "0           0           0           0           0           0           0   \n",
       "1           0           0           0           0           0           0   \n",
       "2           0           0           0           0           0           0   \n",
       "3           0           0           0           0           0           0   \n",
       "4           0           0           0           0           0           0   \n",
       "\n",
       "   z9svx3mA4s  zUlk9c7UEv  zfYl0YECr0  \n",
       "0           0           0           0  \n",
       "1           0           0           0  \n",
       "2           0           0           0  \n",
       "3           0           0           0  \n",
       "4           0           0           0  \n",
       "\n",
       "[5 rows x 174 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Student ID\n",
    "sid_features = one_dim_split(df, \"Anon Student Id\")\n",
    "sid_features.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Two-dimensional Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def join_columns(row, cols):\n",
    "    return \",\".join([str(row[col]) for col in cols])\n",
    "def joint_name(cols):\n",
    "    return \",\".join(cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column Anon Student Id,Problem Unit has 1130 values, so we will add as many columns to this dataframe.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_01</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_02</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_04</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_06</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_08</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_10</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_12</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_13</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_14</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_15</th>\n",
       "      <th>...</th>\n",
       "      <th>zUlk9c7UEv,Unit CTA1_08</th>\n",
       "      <th>zUlk9c7UEv,Unit CTA1_10</th>\n",
       "      <th>zUlk9c7UEv,Unit ES_03</th>\n",
       "      <th>zUlk9c7UEv,Unit ES_04</th>\n",
       "      <th>zfYl0YECr0,Unit CTA1_02</th>\n",
       "      <th>zfYl0YECr0,Unit CTA1_04</th>\n",
       "      <th>zfYl0YECr0,Unit CTA1_06</th>\n",
       "      <th>zfYl0YECr0,Unit CTA1_13</th>\n",
       "      <th>zfYl0YECr0,Unit ES_01</th>\n",
       "      <th>zfYl0YECr0,Unit ES_02</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1130 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   02i5jCrfQK,Unit CTA1_01  02i5jCrfQK,Unit CTA1_02  02i5jCrfQK,Unit CTA1_04  \\\n",
       "0                        0                        0                        0   \n",
       "1                        0                        0                        0   \n",
       "2                        0                        0                        0   \n",
       "3                        0                        0                        0   \n",
       "4                        0                        0                        0   \n",
       "\n",
       "   02i5jCrfQK,Unit CTA1_06  02i5jCrfQK,Unit CTA1_08  02i5jCrfQK,Unit CTA1_10  \\\n",
       "0                        0                        0                        0   \n",
       "1                        0                        0                        0   \n",
       "2                        0                        0                        0   \n",
       "3                        0                        0                        0   \n",
       "4                        0                        0                        0   \n",
       "\n",
       "   02i5jCrfQK,Unit CTA1_12  02i5jCrfQK,Unit CTA1_13  02i5jCrfQK,Unit CTA1_14  \\\n",
       "0                        0                        0                        0   \n",
       "1                        0                        0                        0   \n",
       "2                        0                        0                        0   \n",
       "3                        0                        0                        0   \n",
       "4                        0                        0                        0   \n",
       "\n",
       "   02i5jCrfQK,Unit CTA1_15  ...  zUlk9c7UEv,Unit CTA1_08  \\\n",
       "0                        0  ...                        0   \n",
       "1                        0  ...                        0   \n",
       "2                        0  ...                        0   \n",
       "3                        0  ...                        0   \n",
       "4                        0  ...                        0   \n",
       "\n",
       "   zUlk9c7UEv,Unit CTA1_10  zUlk9c7UEv,Unit ES_03  zUlk9c7UEv,Unit ES_04  \\\n",
       "0                        0                      0                      0   \n",
       "1                        0                      0                      0   \n",
       "2                        0                      0                      0   \n",
       "3                        0                      0                      0   \n",
       "4                        0                      0                      0   \n",
       "\n",
       "   zfYl0YECr0,Unit CTA1_02  zfYl0YECr0,Unit CTA1_04  zfYl0YECr0,Unit CTA1_06  \\\n",
       "0                        0                        0                        0   \n",
       "1                        0                        0                        0   \n",
       "2                        0                        0                        0   \n",
       "3                        0                        0                        0   \n",
       "4                        0                        0                        0   \n",
       "\n",
       "   zfYl0YECr0,Unit CTA1_13  zfYl0YECr0,Unit ES_01  zfYl0YECr0,Unit ES_02  \n",
       "0                        0                      0                      0  \n",
       "1                        0                      0                      0  \n",
       "2                        0                      0                      0  \n",
       "3                        0                      0                      0  \n",
       "4                        0                      0                      0  \n",
       "\n",
       "[5 rows x 1130 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Student ID and Problem Unit\n",
    "cols = [\"Anon Student Id\", \"Problem Unit\"]\n",
    "df[joint_name(cols)] = df.apply(join_columns, axis=1, args=(cols,))\n",
    "sid_pu_features = one_dim_split(df, joint_name(cols))\n",
    "sid_pu_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column Problem Unit,Problem Section has 138 values, so we will add as many columns to this dataframe.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-1</th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-2</th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-3</th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-4</th>\n",
       "      <th>Unit CTA1_02,Section CTA1_02-1</th>\n",
       "      <th>Unit CTA1_02,Section CTA1_02-2</th>\n",
       "      <th>Unit CTA1_02,Section CTA1_02-3</th>\n",
       "      <th>Unit CTA1_02,Section CTA1_02-4</th>\n",
       "      <th>Unit CTA1_04,Section CTA1_04-1</th>\n",
       "      <th>Unit CTA1_04,Section CTA1_04-2</th>\n",
       "      <th>...</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans10</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans11</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans2</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans3</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans4</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans5</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans6</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans7</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans8</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 138 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unit CTA1_01,Section CTA1_01-1  Unit CTA1_01,Section CTA1_01-2  \\\n",
       "0                               0                               0   \n",
       "1                               0                               0   \n",
       "2                               0                               0   \n",
       "3                               0                               0   \n",
       "4                               0                               0   \n",
       "\n",
       "   Unit CTA1_01,Section CTA1_01-3  Unit CTA1_01,Section CTA1_01-4  \\\n",
       "0                               0                               0   \n",
       "1                               0                               0   \n",
       "2                               0                               0   \n",
       "3                               0                               0   \n",
       "4                               0                               0   \n",
       "\n",
       "   Unit CTA1_02,Section CTA1_02-1  Unit CTA1_02,Section CTA1_02-2  \\\n",
       "0                               0                               0   \n",
       "1                               0                               0   \n",
       "2                               0                               0   \n",
       "3                               0                               0   \n",
       "4                               0                               0   \n",
       "\n",
       "   Unit CTA1_02,Section CTA1_02-3  Unit CTA1_02,Section CTA1_02-4  \\\n",
       "0                               0                               0   \n",
       "1                               0                               0   \n",
       "2                               0                               0   \n",
       "3                               0                               0   \n",
       "4                               0                               0   \n",
       "\n",
       "   Unit CTA1_04,Section CTA1_04-1  Unit CTA1_04,Section CTA1_04-2  ...  \\\n",
       "0                               0                               0  ...   \n",
       "1                               0                               0  ...   \n",
       "2                               0                               0  ...   \n",
       "3                               0                               0  ...   \n",
       "4                               0                               0  ...   \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans10  \\\n",
       "0                                               0   \n",
       "1                                               0   \n",
       "2                                               0   \n",
       "3                                               0   \n",
       "4                                               0   \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans11  \\\n",
       "0                                               0   \n",
       "1                                               0   \n",
       "2                                               0   \n",
       "3                                               0   \n",
       "4                                               0   \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans2  \\\n",
       "0                                              0   \n",
       "1                                              0   \n",
       "2                                              0   \n",
       "3                                              0   \n",
       "4                                              0   \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans3  \\\n",
       "0                                              0   \n",
       "1                                              0   \n",
       "2                                              0   \n",
       "3                                              0   \n",
       "4                                              0   \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans4  \\\n",
       "0                                              0   \n",
       "1                                              0   \n",
       "2                                              0   \n",
       "3                                              0   \n",
       "4                                              0   \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans5  \\\n",
       "0                                              0   \n",
       "1                                              0   \n",
       "2                                              0   \n",
       "3                                              0   \n",
       "4                                              0   \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans6  \\\n",
       "0                                              0   \n",
       "1                                              0   \n",
       "2                                              0   \n",
       "3                                              0   \n",
       "4                                              0   \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans7  \\\n",
       "0                                              0   \n",
       "1                                              0   \n",
       "2                                              0   \n",
       "3                                              0   \n",
       "4                                              0   \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans8  \\\n",
       "0                                              0   \n",
       "1                                              0   \n",
       "2                                              0   \n",
       "3                                              0   \n",
       "4                                              0   \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans9  \n",
       "0                                              0  \n",
       "1                                              0  \n",
       "2                                              0  \n",
       "3                                              0  \n",
       "4                                              0  \n",
       "\n",
       "[5 rows x 138 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Problem Unit and Problem Section\n",
    "cols = [\"Problem Unit\", \"Problem Section\"]\n",
    "df[joint_name(cols)] = df.apply(join_columns, axis=1, args=(cols,))\n",
    "pu_ps_features = one_dim_split(df, joint_name(cols))\n",
    "pu_ps_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column Problem Section,Problem Name has 1122 values, so we will add as many columns to this dataframe.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Section CTA1_01-1,LDEMO_WKST</th>\n",
       "      <th>Section CTA1_01-2,BH1T08</th>\n",
       "      <th>Section CTA1_01-2,BH1T08A</th>\n",
       "      <th>Section CTA1_01-2,BH1T08B</th>\n",
       "      <th>Section CTA1_01-2,BH1T08C</th>\n",
       "      <th>Section CTA1_01-2,BH1T15</th>\n",
       "      <th>Section CTA1_01-2,BH1T15A</th>\n",
       "      <th>Section CTA1_01-2,BH1T15B</th>\n",
       "      <th>Section CTA1_01-2,BH1T15C</th>\n",
       "      <th>Section CTA1_01-2,BH1T17</th>\n",
       "      <th>...</th>\n",
       "      <th>Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR21</th>\n",
       "      <th>Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR22</th>\n",
       "      <th>Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR23</th>\n",
       "      <th>Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR24</th>\n",
       "      <th>Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR25</th>\n",
       "      <th>Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR26</th>\n",
       "      <th>Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR27</th>\n",
       "      <th>Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR28</th>\n",
       "      <th>Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR29</th>\n",
       "      <th>Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR30</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1122 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Section CTA1_01-1,LDEMO_WKST  Section CTA1_01-2,BH1T08  \\\n",
       "0                             0                         0   \n",
       "1                             0                         0   \n",
       "2                             0                         0   \n",
       "3                             0                         0   \n",
       "4                             0                         0   \n",
       "\n",
       "   Section CTA1_01-2,BH1T08A  Section CTA1_01-2,BH1T08B  \\\n",
       "0                          0                          0   \n",
       "1                          0                          0   \n",
       "2                          0                          0   \n",
       "3                          0                          0   \n",
       "4                          0                          0   \n",
       "\n",
       "   Section CTA1_01-2,BH1T08C  Section CTA1_01-2,BH1T15  \\\n",
       "0                          0                         0   \n",
       "1                          0                         0   \n",
       "2                          0                         0   \n",
       "3                          0                         0   \n",
       "4                          0                         0   \n",
       "\n",
       "   Section CTA1_01-2,BH1T15A  Section CTA1_01-2,BH1T15B  \\\n",
       "0                          0                          0   \n",
       "1                          0                          0   \n",
       "2                          0                          0   \n",
       "3                          0                          0   \n",
       "4                          0                          0   \n",
       "\n",
       "   Section CTA1_01-2,BH1T15C  Section CTA1_01-2,BH1T17  ...  \\\n",
       "0                          0                         0  ...   \n",
       "1                          0                         0  ...   \n",
       "2                          0                         0  ...   \n",
       "3                          0                         0  ...   \n",
       "4                          0                         0  ...   \n",
       "\n",
       "   Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR21  \\\n",
       "0                                              0   \n",
       "1                                              0   \n",
       "2                                              0   \n",
       "3                                              0   \n",
       "4                                              0   \n",
       "\n",
       "   Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR22  \\\n",
       "0                                              0   \n",
       "1                                              0   \n",
       "2                                              0   \n",
       "3                                              0   \n",
       "4                                              0   \n",
       "\n",
       "   Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR23  \\\n",
       "0                                              0   \n",
       "1                                              0   \n",
       "2                                              0   \n",
       "3                                              0   \n",
       "4                                              0   \n",
       "\n",
       "   Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR24  \\\n",
       "0                                              0   \n",
       "1                                              0   \n",
       "2                                              0   \n",
       "3                                              0   \n",
       "4                                              0   \n",
       "\n",
       "   Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR25  \\\n",
       "0                                              0   \n",
       "1                                              0   \n",
       "2                                              0   \n",
       "3                                              0   \n",
       "4                                              0   \n",
       "\n",
       "   Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR26  \\\n",
       "0                                              0   \n",
       "1                                              0   \n",
       "2                                              0   \n",
       "3                                              0   \n",
       "4                                              0   \n",
       "\n",
       "   Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR27  \\\n",
       "0                                              0   \n",
       "1                                              0   \n",
       "2                                              0   \n",
       "3                                              0   \n",
       "4                                              0   \n",
       "\n",
       "   Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR28  \\\n",
       "0                                              0   \n",
       "1                                              0   \n",
       "2                                              0   \n",
       "3                                              0   \n",
       "4                                              0   \n",
       "\n",
       "   Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR29  \\\n",
       "0                                              0   \n",
       "1                                              0   \n",
       "2                                              0   \n",
       "3                                              0   \n",
       "4                                              0   \n",
       "\n",
       "   Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR30  \n",
       "0                                              0  \n",
       "1                                              0  \n",
       "2                                              0  \n",
       "3                                              0  \n",
       "4                                              0  \n",
       "\n",
       "[5 rows x 1122 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Problem Section and Problem Name\n",
    "cols = [\"Problem Section\", \"Problem Name\"]\n",
    "df[joint_name(cols)] = df.apply(join_columns, axis=1, args=(cols,))\n",
    "ps_pn_features = one_dim_split(df, joint_name(cols))\n",
    "ps_pn_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column Problem Name,Step Name has 74835 values, so we will add as many columns to this dataframe.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1PTB02,-160 = -40X</th>\n",
       "      <th>1PTB02,-400 = -40X</th>\n",
       "      <th>1PTB02,-40A = -160</th>\n",
       "      <th>1PTB02,-40A = -400</th>\n",
       "      <th>1PTB02,-40A+400 = 0</th>\n",
       "      <th>1PTB02,-40A+400 = 240</th>\n",
       "      <th>1PTB02,-40B = -160</th>\n",
       "      <th>1PTB02,-40B = -400</th>\n",
       "      <th>1PTB02,-40B+400 = 0</th>\n",
       "      <th>1PTB02,-40B+400 = 240</th>\n",
       "      <th>...</th>\n",
       "      <th>YLT3X+5,XIntercept1</th>\n",
       "      <th>YLT3X+5,YCoordinate1_1</th>\n",
       "      <th>YLT3X+5,YCoordinate1_2</th>\n",
       "      <th>YLT3X+5,YIntercept1</th>\n",
       "      <th>YLT3X+5,xMax</th>\n",
       "      <th>YLT3X+5,xMin</th>\n",
       "      <th>YLT3X+5,y = 0+5</th>\n",
       "      <th>YLT3X+5,y = 3*0+5</th>\n",
       "      <th>YLT3X+5,yMax</th>\n",
       "      <th>YLT3X+5,yMin</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 74835 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   1PTB02,-160 = -40X  1PTB02,-400 = -40X  1PTB02,-40A = -160  \\\n",
       "0                   0                   0                   0   \n",
       "1                   0                   0                   0   \n",
       "2                   0                   0                   0   \n",
       "3                   0                   0                   0   \n",
       "4                   0                   0                   0   \n",
       "\n",
       "   1PTB02,-40A = -400  1PTB02,-40A+400 = 0  1PTB02,-40A+400 = 240  \\\n",
       "0                   0                    0                      0   \n",
       "1                   0                    0                      0   \n",
       "2                   0                    0                      0   \n",
       "3                   0                    0                      0   \n",
       "4                   0                    0                      0   \n",
       "\n",
       "   1PTB02,-40B = -160  1PTB02,-40B = -400  1PTB02,-40B+400 = 0  \\\n",
       "0                   0                   0                    0   \n",
       "1                   0                   0                    0   \n",
       "2                   0                   0                    0   \n",
       "3                   0                   0                    0   \n",
       "4                   0                   0                    0   \n",
       "\n",
       "   1PTB02,-40B+400 = 240  ...  YLT3X+5,XIntercept1  YLT3X+5,YCoordinate1_1  \\\n",
       "0                      0  ...                    0                       0   \n",
       "1                      0  ...                    0                       0   \n",
       "2                      0  ...                    0                       0   \n",
       "3                      0  ...                    0                       0   \n",
       "4                      0  ...                    0                       0   \n",
       "\n",
       "   YLT3X+5,YCoordinate1_2  YLT3X+5,YIntercept1  YLT3X+5,xMax  YLT3X+5,xMin  \\\n",
       "0                       0                    0             0             0   \n",
       "1                       0                    0             0             0   \n",
       "2                       0                    0             0             0   \n",
       "3                       0                    0             0             0   \n",
       "4                       0                    0             0             0   \n",
       "\n",
       "   YLT3X+5,y = 0+5  YLT3X+5,y = 3*0+5  YLT3X+5,yMax  YLT3X+5,yMin  \n",
       "0                0                  0             0             0  \n",
       "1                0                  0             0             0  \n",
       "2                0                  0             0             0  \n",
       "3                0                  0             0             0  \n",
       "4                0                  0             0             0  \n",
       "\n",
       "[5 rows x 74835 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Problem Name and Step Name\n",
    "cols = [\"Problem Name\", \"Step Name\"]\n",
    "df[joint_name(cols)] = df.apply(join_columns, axis=1, args=(cols,))\n",
    "pn_sn_features = one_dim_split(df, joint_name(cols))\n",
    "pn_sn_features.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hyper-dimensional Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column Anon Student Id,Problem Name,Problem Section has 18699 values, so we will add as many columns to this dataframe.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>02i5jCrfQK,1PTB02,Section CTA1_14-2</th>\n",
       "      <th>02i5jCrfQK,1PTB03,Section CTA1_14-2</th>\n",
       "      <th>02i5jCrfQK,1PTB05,Section CTA1_20-1</th>\n",
       "      <th>02i5jCrfQK,1PTB07,Section CTA1_14-2</th>\n",
       "      <th>02i5jCrfQK,1PTB08,Section CTA1_20-2</th>\n",
       "      <th>02i5jCrfQK,1PTB09,Section CTA1_14-2</th>\n",
       "      <th>02i5jCrfQK,1PTFB10,Section CTA1_20-2</th>\n",
       "      <th>02i5jCrfQK,1PTFB11,Section CTA1_14-2</th>\n",
       "      <th>02i5jCrfQK,1PTFB13,Section CTA1_14-2</th>\n",
       "      <th>02i5jCrfQK,1PTFB14,Section CTA1_14-2</th>\n",
       "      <th>...</th>\n",
       "      <th>zfYl0YECr0,PERCENT01C,Section CTA1_04-2</th>\n",
       "      <th>zfYl0YECr0,PROP03,Section CTA1_13-1</th>\n",
       "      <th>zfYl0YECr0,PROP04,Section CTA1_13-1</th>\n",
       "      <th>zfYl0YECr0,PROP05,Section CTA1_13-1</th>\n",
       "      <th>zfYl0YECr0,PROP07,Section CTA1_13-1</th>\n",
       "      <th>zfYl0YECr0,PROP08,Section CTA1_13-1</th>\n",
       "      <th>zfYl0YECr0,PROP12,Section CTA1_13-1</th>\n",
       "      <th>zfYl0YECr0,REAL13C,Section CTA1_04-2</th>\n",
       "      <th>zfYl0YECr0,REAL19B,Section CTA1_02-4</th>\n",
       "      <th>zfYl0YECr0,REAL28,Section CTA1_02-4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 18699 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   02i5jCrfQK,1PTB02,Section CTA1_14-2  02i5jCrfQK,1PTB03,Section CTA1_14-2  \\\n",
       "0                                    0                                    0   \n",
       "1                                    0                                    0   \n",
       "2                                    0                                    0   \n",
       "3                                    0                                    0   \n",
       "4                                    0                                    0   \n",
       "\n",
       "   02i5jCrfQK,1PTB05,Section CTA1_20-1  02i5jCrfQK,1PTB07,Section CTA1_14-2  \\\n",
       "0                                    0                                    0   \n",
       "1                                    0                                    0   \n",
       "2                                    0                                    0   \n",
       "3                                    0                                    0   \n",
       "4                                    0                                    0   \n",
       "\n",
       "   02i5jCrfQK,1PTB08,Section CTA1_20-2  02i5jCrfQK,1PTB09,Section CTA1_14-2  \\\n",
       "0                                    0                                    0   \n",
       "1                                    0                                    0   \n",
       "2                                    0                                    0   \n",
       "3                                    0                                    0   \n",
       "4                                    0                                    0   \n",
       "\n",
       "   02i5jCrfQK,1PTFB10,Section CTA1_20-2  02i5jCrfQK,1PTFB11,Section CTA1_14-2  \\\n",
       "0                                     0                                     0   \n",
       "1                                     0                                     0   \n",
       "2                                     0                                     0   \n",
       "3                                     0                                     0   \n",
       "4                                     0                                     0   \n",
       "\n",
       "   02i5jCrfQK,1PTFB13,Section CTA1_14-2  02i5jCrfQK,1PTFB14,Section CTA1_14-2  \\\n",
       "0                                     0                                     0   \n",
       "1                                     0                                     0   \n",
       "2                                     0                                     0   \n",
       "3                                     0                                     0   \n",
       "4                                     0                                     0   \n",
       "\n",
       "   ...  zfYl0YECr0,PERCENT01C,Section CTA1_04-2  \\\n",
       "0  ...                                        0   \n",
       "1  ...                                        0   \n",
       "2  ...                                        0   \n",
       "3  ...                                        0   \n",
       "4  ...                                        0   \n",
       "\n",
       "   zfYl0YECr0,PROP03,Section CTA1_13-1  zfYl0YECr0,PROP04,Section CTA1_13-1  \\\n",
       "0                                    0                                    0   \n",
       "1                                    0                                    0   \n",
       "2                                    0                                    0   \n",
       "3                                    0                                    0   \n",
       "4                                    0                                    0   \n",
       "\n",
       "   zfYl0YECr0,PROP05,Section CTA1_13-1  zfYl0YECr0,PROP07,Section CTA1_13-1  \\\n",
       "0                                    0                                    0   \n",
       "1                                    0                                    0   \n",
       "2                                    0                                    0   \n",
       "3                                    0                                    0   \n",
       "4                                    0                                    0   \n",
       "\n",
       "   zfYl0YECr0,PROP08,Section CTA1_13-1  zfYl0YECr0,PROP12,Section CTA1_13-1  \\\n",
       "0                                    0                                    0   \n",
       "1                                    0                                    0   \n",
       "2                                    0                                    0   \n",
       "3                                    0                                    0   \n",
       "4                                    0                                    0   \n",
       "\n",
       "   zfYl0YECr0,REAL13C,Section CTA1_04-2  zfYl0YECr0,REAL19B,Section CTA1_02-4  \\\n",
       "0                                     0                                     0   \n",
       "1                                     0                                     0   \n",
       "2                                     0                                     0   \n",
       "3                                     0                                     0   \n",
       "4                                     0                                     0   \n",
       "\n",
       "   zfYl0YECr0,REAL28,Section CTA1_02-4  \n",
       "0                                    0  \n",
       "1                                    0  \n",
       "2                                    0  \n",
       "3                                    0  \n",
       "4                                    0  \n",
       "\n",
       "[5 rows x 18699 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Student Name and Problem Name and Section Name\n",
    "cols = [\"Anon Student Id\", \"Problem Name\", \"Problem Section\"]\n",
    "df[joint_name(cols)] = df.apply(join_columns, axis=1, args=(cols,))\n",
    "sid_pu_ps_features = one_dim_split(df, joint_name(cols))\n",
    "sid_pu_ps_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column Problem Unit,Problem Section,Problem Name has 1122 values, so we will add as many columns to this dataframe.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST</th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-2,BH1T08</th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-2,BH1T08A</th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-2,BH1T08B</th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-2,BH1T08C</th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-2,BH1T15</th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-2,BH1T15A</th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-2,BH1T15B</th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-2,BH1T15C</th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-2,BH1T17</th>\n",
       "      <th>...</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN007</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN008</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN009</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN010</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN011</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN012</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN013</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN014</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN015</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN016</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1122 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST  \\\n",
       "0                                          0   \n",
       "1                                          0   \n",
       "2                                          0   \n",
       "3                                          0   \n",
       "4                                          0   \n",
       "\n",
       "   Unit CTA1_01,Section CTA1_01-2,BH1T08  \\\n",
       "0                                      0   \n",
       "1                                      0   \n",
       "2                                      0   \n",
       "3                                      0   \n",
       "4                                      0   \n",
       "\n",
       "   Unit CTA1_01,Section CTA1_01-2,BH1T08A  \\\n",
       "0                                       0   \n",
       "1                                       0   \n",
       "2                                       0   \n",
       "3                                       0   \n",
       "4                                       0   \n",
       "\n",
       "   Unit CTA1_01,Section CTA1_01-2,BH1T08B  \\\n",
       "0                                       0   \n",
       "1                                       0   \n",
       "2                                       0   \n",
       "3                                       0   \n",
       "4                                       0   \n",
       "\n",
       "   Unit CTA1_01,Section CTA1_01-2,BH1T08C  \\\n",
       "0                                       0   \n",
       "1                                       0   \n",
       "2                                       0   \n",
       "3                                       0   \n",
       "4                                       0   \n",
       "\n",
       "   Unit CTA1_01,Section CTA1_01-2,BH1T15  \\\n",
       "0                                      0   \n",
       "1                                      0   \n",
       "2                                      0   \n",
       "3                                      0   \n",
       "4                                      0   \n",
       "\n",
       "   Unit CTA1_01,Section CTA1_01-2,BH1T15A  \\\n",
       "0                                       0   \n",
       "1                                       0   \n",
       "2                                       0   \n",
       "3                                       0   \n",
       "4                                       0   \n",
       "\n",
       "   Unit CTA1_01,Section CTA1_01-2,BH1T15B  \\\n",
       "0                                       0   \n",
       "1                                       0   \n",
       "2                                       0   \n",
       "3                                       0   \n",
       "4                                       0   \n",
       "\n",
       "   Unit CTA1_01,Section CTA1_01-2,BH1T15C  \\\n",
       "0                                       0   \n",
       "1                                       0   \n",
       "2                                       0   \n",
       "3                                       0   \n",
       "4                                       0   \n",
       "\n",
       "   Unit CTA1_01,Section CTA1_01-2,BH1T17  ...  \\\n",
       "0                                      0  ...   \n",
       "1                                      0  ...   \n",
       "2                                      0  ...   \n",
       "3                                      0  ...   \n",
       "4                                      0  ...   \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN007  \\\n",
       "0                                                  0             \n",
       "1                                                  0             \n",
       "2                                                  0             \n",
       "3                                                  0             \n",
       "4                                                  0             \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN008  \\\n",
       "0                                                  0             \n",
       "1                                                  0             \n",
       "2                                                  0             \n",
       "3                                                  0             \n",
       "4                                                  0             \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN009  \\\n",
       "0                                                  0             \n",
       "1                                                  0             \n",
       "2                                                  0             \n",
       "3                                                  0             \n",
       "4                                                  0             \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN010  \\\n",
       "0                                                  0             \n",
       "1                                                  0             \n",
       "2                                                  0             \n",
       "3                                                  0             \n",
       "4                                                  0             \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN011  \\\n",
       "0                                                  0             \n",
       "1                                                  0             \n",
       "2                                                  0             \n",
       "3                                                  0             \n",
       "4                                                  0             \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN012  \\\n",
       "0                                                  0             \n",
       "1                                                  0             \n",
       "2                                                  0             \n",
       "3                                                  0             \n",
       "4                                                  0             \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN013  \\\n",
       "0                                                  0             \n",
       "1                                                  0             \n",
       "2                                                  0             \n",
       "3                                                  0             \n",
       "4                                                  0             \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN014  \\\n",
       "0                                                  0             \n",
       "1                                                  0             \n",
       "2                                                  0             \n",
       "3                                                  0             \n",
       "4                                                  0             \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN015  \\\n",
       "0                                                  0             \n",
       "1                                                  0             \n",
       "2                                                  0             \n",
       "3                                                  0             \n",
       "4                                                  0             \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN016  \n",
       "0                                                  0            \n",
       "1                                                  0            \n",
       "2                                                  0            \n",
       "3                                                  0            \n",
       "4                                                  0            \n",
       "\n",
       "[5 rows x 1122 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Problem Unit and Problem Section and Problem Name\n",
    "cols = [\"Problem Unit\", \"Problem Section\", \"Problem Name\"]\n",
    "df[joint_name(cols)] = df.apply(join_columns, axis=1, args=(cols,))\n",
    "pu_ps_pn_features = one_dim_split(df, joint_name(cols))\n",
    "pu_ps_pn_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column Problem Section,Problem Name,Step Name has 75405 values, so we will add as many columns to this dataframe.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Section CTA1_01-1,LDEMO_WKST,R1C1</th>\n",
       "      <th>Section CTA1_01-1,LDEMO_WKST,R1C2</th>\n",
       "      <th>Section CTA1_01-1,LDEMO_WKST,R2C1</th>\n",
       "      <th>Section CTA1_01-1,LDEMO_WKST,R2C2</th>\n",
       "      <th>Section CTA1_01-1,LDEMO_WKST,R3C1</th>\n",
       "      <th>Section CTA1_01-1,LDEMO_WKST,R3C2</th>\n",
       "      <th>Section CTA1_01-1,LDEMO_WKST,R4C1</th>\n",
       "      <th>Section CTA1_01-1,LDEMO_WKST,R4C2</th>\n",
       "      <th>Section CTA1_01-1,LDEMO_WKST,R5C1</th>\n",
       "      <th>Section CTA1_01-1,LDEMO_WKST,R5C2</th>\n",
       "      <th>...</th>\n",
       "      <th>Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR30,-2.38516481/10 = x</th>\n",
       "      <th>Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR30,5x^2-3x = 1</th>\n",
       "      <th>Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR30,5x^2-3x-1 = 0</th>\n",
       "      <th>Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR30,8.38516481/10 = x</th>\n",
       "      <th>Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR30,a-factor-node</th>\n",
       "      <th>Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR30,b-factor-node</th>\n",
       "      <th>Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR30,c-factor-node</th>\n",
       "      <th>Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR30,den1-factor-node</th>\n",
       "      <th>Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR30,num1-factor-node</th>\n",
       "      <th>Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR30,num2-factor-node</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 75405 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Section CTA1_01-1,LDEMO_WKST,R1C1  Section CTA1_01-1,LDEMO_WKST,R1C2  \\\n",
       "0                                  0                                  0   \n",
       "1                                  0                                  0   \n",
       "2                                  0                                  0   \n",
       "3                                  0                                  0   \n",
       "4                                  0                                  0   \n",
       "\n",
       "   Section CTA1_01-1,LDEMO_WKST,R2C1  Section CTA1_01-1,LDEMO_WKST,R2C2  \\\n",
       "0                                  0                                  0   \n",
       "1                                  0                                  0   \n",
       "2                                  0                                  0   \n",
       "3                                  0                                  0   \n",
       "4                                  0                                  0   \n",
       "\n",
       "   Section CTA1_01-1,LDEMO_WKST,R3C1  Section CTA1_01-1,LDEMO_WKST,R3C2  \\\n",
       "0                                  0                                  0   \n",
       "1                                  0                                  0   \n",
       "2                                  0                                  0   \n",
       "3                                  0                                  0   \n",
       "4                                  0                                  0   \n",
       "\n",
       "   Section CTA1_01-1,LDEMO_WKST,R4C1  Section CTA1_01-1,LDEMO_WKST,R4C2  \\\n",
       "0                                  0                                  0   \n",
       "1                                  0                                  0   \n",
       "2                                  0                                  0   \n",
       "3                                  0                                  0   \n",
       "4                                  0                                  0   \n",
       "\n",
       "   Section CTA1_01-1,LDEMO_WKST,R5C1  Section CTA1_01-1,LDEMO_WKST,R5C2  ...  \\\n",
       "0                                  0                                  0  ...   \n",
       "1                                  0                                  0  ...   \n",
       "2                                  0                                  0  ...   \n",
       "3                                  0                                  0  ...   \n",
       "4                                  0                                  0  ...   \n",
       "\n",
       "   Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR30,-2.38516481/10 = x  \\\n",
       "0                                                  0                  \n",
       "1                                                  0                  \n",
       "2                                                  0                  \n",
       "3                                                  0                  \n",
       "4                                                  0                  \n",
       "\n",
       "   Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR30,5x^2-3x = 1  \\\n",
       "0                                                  0           \n",
       "1                                                  0           \n",
       "2                                                  0           \n",
       "3                                                  0           \n",
       "4                                                  0           \n",
       "\n",
       "   Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR30,5x^2-3x-1 = 0  \\\n",
       "0                                                  0             \n",
       "1                                                  0             \n",
       "2                                                  0             \n",
       "3                                                  0             \n",
       "4                                                  0             \n",
       "\n",
       "   Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR30,8.38516481/10 = x  \\\n",
       "0                                                  0                 \n",
       "1                                                  0                 \n",
       "2                                                  0                 \n",
       "3                                                  0                 \n",
       "4                                                  0                 \n",
       "\n",
       "   Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR30,a-factor-node  \\\n",
       "0                                                  0             \n",
       "1                                                  0             \n",
       "2                                                  0             \n",
       "3                                                  0             \n",
       "4                                                  0             \n",
       "\n",
       "   Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR30,b-factor-node  \\\n",
       "0                                                  0             \n",
       "1                                                  0             \n",
       "2                                                  0             \n",
       "3                                                  0             \n",
       "4                                                  0             \n",
       "\n",
       "   Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR30,c-factor-node  \\\n",
       "0                                                  0             \n",
       "1                                                  0             \n",
       "2                                                  0             \n",
       "3                                                  0             \n",
       "4                                                  0             \n",
       "\n",
       "   Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR30,den1-factor-node  \\\n",
       "0                                                  0                \n",
       "1                                                  0                \n",
       "2                                                  0                \n",
       "3                                                  0                \n",
       "4                                                  0                \n",
       "\n",
       "   Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR30,num1-factor-node  \\\n",
       "0                                                  0                \n",
       "1                                                  0                \n",
       "2                                                  0                \n",
       "3                                                  0                \n",
       "4                                                  0                \n",
       "\n",
       "   Section QUADRATICS-SOLVING_ES-1,EG-S-FACTOR30,num2-factor-node  \n",
       "0                                                  0               \n",
       "1                                                  0               \n",
       "2                                                  0               \n",
       "3                                                  0               \n",
       "4                                                  0               \n",
       "\n",
       "[5 rows x 75405 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Problem Section and Problem Name and Step Name\n",
    "cols = [\"Problem Section\", \"Problem Name\", \"Step Name\"]\n",
    "df[joint_name(cols)] = df.apply(join_columns, axis=1, args=(cols,))\n",
    "ps_pn_sn_features = one_dim_split(df, joint_name(cols))\n",
    "ps_pn_sn_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column Anon Student Id,Problem Unit,Problem Section,Problem Name has 18699 values, so we will add as many columns to this dataframe.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_01,Section CTA1_01-2,BH1T08</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_01,Section CTA1_01-2,BH1T17C</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_01,Section CTA1_01-2,BH1T22A</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_01,Section CTA1_01-2,BH1T24</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_01,Section CTA1_01-2,BH1T26A</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_01,Section CTA1_01-2,RXMX1</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_01,Section CTA1_01-3,BH1T19A</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_01,Section CTA1_01-3,BH1T27B</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_01,Section CTA1_01-3,BH1T28C</th>\n",
       "      <th>...</th>\n",
       "      <th>zfYl0YECr0,Unit ES_02,Section ES_02-3,EG34A</th>\n",
       "      <th>zfYl0YECr0,Unit ES_02,Section ES_02-3,EG35A</th>\n",
       "      <th>zfYl0YECr0,Unit ES_02,Section ES_02-3,EG35B</th>\n",
       "      <th>zfYl0YECr0,Unit ES_02,Section ES_02-4,EG34A</th>\n",
       "      <th>zfYl0YECr0,Unit ES_02,Section ES_02-4,EG35B</th>\n",
       "      <th>zfYl0YECr0,Unit ES_02,Section ES_02-4,EG35C</th>\n",
       "      <th>zfYl0YECr0,Unit ES_02,Section ES_02-5,EG40</th>\n",
       "      <th>zfYl0YECr0,Unit ES_02,Section ES_02-6,EG40</th>\n",
       "      <th>zfYl0YECr0,Unit ES_02,Section ES_02-7,EG41</th>\n",
       "      <th>zfYl0YECr0,Unit ES_02,Section ES_02-8,EG41</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 18699 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   02i5jCrfQK,Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST  \\\n",
       "0                                                  0      \n",
       "1                                                  0      \n",
       "2                                                  0      \n",
       "3                                                  0      \n",
       "4                                                  0      \n",
       "\n",
       "   02i5jCrfQK,Unit CTA1_01,Section CTA1_01-2,BH1T08  \\\n",
       "0                                                 0   \n",
       "1                                                 0   \n",
       "2                                                 0   \n",
       "3                                                 0   \n",
       "4                                                 0   \n",
       "\n",
       "   02i5jCrfQK,Unit CTA1_01,Section CTA1_01-2,BH1T17C  \\\n",
       "0                                                  0   \n",
       "1                                                  0   \n",
       "2                                                  0   \n",
       "3                                                  0   \n",
       "4                                                  0   \n",
       "\n",
       "   02i5jCrfQK,Unit CTA1_01,Section CTA1_01-2,BH1T22A  \\\n",
       "0                                                  0   \n",
       "1                                                  0   \n",
       "2                                                  0   \n",
       "3                                                  0   \n",
       "4                                                  0   \n",
       "\n",
       "   02i5jCrfQK,Unit CTA1_01,Section CTA1_01-2,BH1T24  \\\n",
       "0                                                 0   \n",
       "1                                                 0   \n",
       "2                                                 0   \n",
       "3                                                 0   \n",
       "4                                                 0   \n",
       "\n",
       "   02i5jCrfQK,Unit CTA1_01,Section CTA1_01-2,BH1T26A  \\\n",
       "0                                                  0   \n",
       "1                                                  0   \n",
       "2                                                  0   \n",
       "3                                                  0   \n",
       "4                                                  0   \n",
       "\n",
       "   02i5jCrfQK,Unit CTA1_01,Section CTA1_01-2,RXMX1  \\\n",
       "0                                                0   \n",
       "1                                                0   \n",
       "2                                                0   \n",
       "3                                                0   \n",
       "4                                                0   \n",
       "\n",
       "   02i5jCrfQK,Unit CTA1_01,Section CTA1_01-3,BH1T19A  \\\n",
       "0                                                  0   \n",
       "1                                                  0   \n",
       "2                                                  0   \n",
       "3                                                  0   \n",
       "4                                                  0   \n",
       "\n",
       "   02i5jCrfQK,Unit CTA1_01,Section CTA1_01-3,BH1T27B  \\\n",
       "0                                                  0   \n",
       "1                                                  0   \n",
       "2                                                  0   \n",
       "3                                                  0   \n",
       "4                                                  0   \n",
       "\n",
       "   02i5jCrfQK,Unit CTA1_01,Section CTA1_01-3,BH1T28C  ...  \\\n",
       "0                                                  0  ...   \n",
       "1                                                  0  ...   \n",
       "2                                                  0  ...   \n",
       "3                                                  0  ...   \n",
       "4                                                  0  ...   \n",
       "\n",
       "   zfYl0YECr0,Unit ES_02,Section ES_02-3,EG34A  \\\n",
       "0                                            0   \n",
       "1                                            0   \n",
       "2                                            0   \n",
       "3                                            0   \n",
       "4                                            0   \n",
       "\n",
       "   zfYl0YECr0,Unit ES_02,Section ES_02-3,EG35A  \\\n",
       "0                                            0   \n",
       "1                                            0   \n",
       "2                                            0   \n",
       "3                                            0   \n",
       "4                                            0   \n",
       "\n",
       "   zfYl0YECr0,Unit ES_02,Section ES_02-3,EG35B  \\\n",
       "0                                            0   \n",
       "1                                            0   \n",
       "2                                            0   \n",
       "3                                            0   \n",
       "4                                            0   \n",
       "\n",
       "   zfYl0YECr0,Unit ES_02,Section ES_02-4,EG34A  \\\n",
       "0                                            0   \n",
       "1                                            0   \n",
       "2                                            0   \n",
       "3                                            0   \n",
       "4                                            0   \n",
       "\n",
       "   zfYl0YECr0,Unit ES_02,Section ES_02-4,EG35B  \\\n",
       "0                                            0   \n",
       "1                                            0   \n",
       "2                                            0   \n",
       "3                                            0   \n",
       "4                                            0   \n",
       "\n",
       "   zfYl0YECr0,Unit ES_02,Section ES_02-4,EG35C  \\\n",
       "0                                            0   \n",
       "1                                            0   \n",
       "2                                            0   \n",
       "3                                            0   \n",
       "4                                            0   \n",
       "\n",
       "   zfYl0YECr0,Unit ES_02,Section ES_02-5,EG40  \\\n",
       "0                                           0   \n",
       "1                                           0   \n",
       "2                                           0   \n",
       "3                                           0   \n",
       "4                                           0   \n",
       "\n",
       "   zfYl0YECr0,Unit ES_02,Section ES_02-6,EG40  \\\n",
       "0                                           0   \n",
       "1                                           0   \n",
       "2                                           0   \n",
       "3                                           0   \n",
       "4                                           0   \n",
       "\n",
       "   zfYl0YECr0,Unit ES_02,Section ES_02-7,EG41  \\\n",
       "0                                           0   \n",
       "1                                           0   \n",
       "2                                           0   \n",
       "3                                           0   \n",
       "4                                           0   \n",
       "\n",
       "   zfYl0YECr0,Unit ES_02,Section ES_02-8,EG41  \n",
       "0                                           0  \n",
       "1                                           0  \n",
       "2                                           0  \n",
       "3                                           0  \n",
       "4                                           0  \n",
       "\n",
       "[5 rows x 18699 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Student Name and Problem Section and Problem Name and Step Name\n",
    "cols = [\"Anon Student Id\", \"Problem Unit\", \"Problem Section\", \"Problem Name\"]\n",
    "df[joint_name(cols)] = df.apply(join_columns, axis=1, args=(cols,))\n",
    "sid_pu_ps_pn_features = one_dim_split(df, joint_name(cols))\n",
    "sid_pu_ps_pn_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column Problem Unit,Problem Section,Problem Name,Step Name has 75405 values, so we will add as many columns to this dataframe.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R1C1</th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R1C2</th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R2C1</th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R2C2</th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R3C1</th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R3C2</th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R4C1</th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R4C2</th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R5C1</th>\n",
       "      <th>Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R5C2</th>\n",
       "      <th>...</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN016,ALGEBRAIC-TRANSFORMATIONS-refl-v</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN016,CurrFunctionText</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN016,GRAPHICAL-TRANSFORMATIONS-a</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN016,GRAPHICAL-TRANSFORMATIONS-h</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN016,GRAPHICAL-TRANSFORMATIONS-refl-v</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN016,ParentCurveChoice</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN016,ParentDescChoice</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN016,ParentFnChoice</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN016,TransformButton-a</th>\n",
       "      <th>Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN016,TransformButton-h</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 75405 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R1C1  \\\n",
       "0                                               0   \n",
       "1                                               0   \n",
       "2                                               0   \n",
       "3                                               0   \n",
       "4                                               0   \n",
       "\n",
       "   Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R1C2  \\\n",
       "0                                               0   \n",
       "1                                               0   \n",
       "2                                               0   \n",
       "3                                               0   \n",
       "4                                               0   \n",
       "\n",
       "   Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R2C1  \\\n",
       "0                                               0   \n",
       "1                                               0   \n",
       "2                                               0   \n",
       "3                                               0   \n",
       "4                                               0   \n",
       "\n",
       "   Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R2C2  \\\n",
       "0                                               0   \n",
       "1                                               0   \n",
       "2                                               0   \n",
       "3                                               0   \n",
       "4                                               0   \n",
       "\n",
       "   Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R3C1  \\\n",
       "0                                               0   \n",
       "1                                               0   \n",
       "2                                               0   \n",
       "3                                               0   \n",
       "4                                               0   \n",
       "\n",
       "   Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R3C2  \\\n",
       "0                                               0   \n",
       "1                                               0   \n",
       "2                                               0   \n",
       "3                                               0   \n",
       "4                                               0   \n",
       "\n",
       "   Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R4C1  \\\n",
       "0                                               0   \n",
       "1                                               0   \n",
       "2                                               0   \n",
       "3                                               0   \n",
       "4                                               0   \n",
       "\n",
       "   Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R4C2  \\\n",
       "0                                               0   \n",
       "1                                               0   \n",
       "2                                               0   \n",
       "3                                               0   \n",
       "4                                               0   \n",
       "\n",
       "   Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R5C1  \\\n",
       "0                                               0   \n",
       "1                                               0   \n",
       "2                                               0   \n",
       "3                                               0   \n",
       "4                                               0   \n",
       "\n",
       "   Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R5C2  ...  \\\n",
       "0                                               0  ...   \n",
       "1                                               0  ...   \n",
       "2                                               0  ...   \n",
       "3                                               0  ...   \n",
       "4                                               0  ...   \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN016,ALGEBRAIC-TRANSFORMATIONS-refl-v  \\\n",
       "0                                                  0                                              \n",
       "1                                                  0                                              \n",
       "2                                                  0                                              \n",
       "3                                                  0                                              \n",
       "4                                                  0                                              \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN016,CurrFunctionText  \\\n",
       "0                                                  0                              \n",
       "1                                                  0                              \n",
       "2                                                  0                              \n",
       "3                                                  0                              \n",
       "4                                                  0                              \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN016,GRAPHICAL-TRANSFORMATIONS-a  \\\n",
       "0                                                  0                                         \n",
       "1                                                  0                                         \n",
       "2                                                  0                                         \n",
       "3                                                  0                                         \n",
       "4                                                  0                                         \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN016,GRAPHICAL-TRANSFORMATIONS-h  \\\n",
       "0                                                  0                                         \n",
       "1                                                  0                                         \n",
       "2                                                  0                                         \n",
       "3                                                  0                                         \n",
       "4                                                  0                                         \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN016,GRAPHICAL-TRANSFORMATIONS-refl-v  \\\n",
       "0                                                  0                                              \n",
       "1                                                  0                                              \n",
       "2                                                  0                                              \n",
       "3                                                  0                                              \n",
       "4                                                  0                                              \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN016,ParentCurveChoice  \\\n",
       "0                                                  0                               \n",
       "1                                                  0                               \n",
       "2                                                  0                               \n",
       "3                                                  0                               \n",
       "4                                                  0                               \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN016,ParentDescChoice  \\\n",
       "0                                                  0                              \n",
       "1                                                  0                              \n",
       "2                                                  0                              \n",
       "3                                                  0                              \n",
       "4                                                  0                              \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN016,ParentFnChoice  \\\n",
       "0                                                  0                            \n",
       "1                                                  0                            \n",
       "2                                                  0                            \n",
       "3                                                  0                            \n",
       "4                                                  0                            \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN016,TransformButton-a  \\\n",
       "0                                                  0                               \n",
       "1                                                  0                               \n",
       "2                                                  0                               \n",
       "3                                                  0                               \n",
       "4                                                  0                               \n",
       "\n",
       "   Unit QuadLinearFuncTrans,Section QLFuncTrans9,TRANSFORMN016,TransformButton-h  \n",
       "0                                                  0                              \n",
       "1                                                  0                              \n",
       "2                                                  0                              \n",
       "3                                                  0                              \n",
       "4                                                  0                              \n",
       "\n",
       "[5 rows x 75405 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Problem Unit and Problem Section and Problem Name and Step Name\n",
    "cols = [\"Problem Unit\", \"Problem Section\", \"Problem Name\", \"Step Name\"]\n",
    "df[joint_name(cols)] = df.apply(join_columns, axis=1, args=(cols,))\n",
    "pu_ps_pn_sn_features = one_dim_split(df, joint_name(cols))\n",
    "pu_ps_pn_sn_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column Anon Student Id,Problem Section has 4760 values, so we will add as many columns to this dataframe.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>02i5jCrfQK,Section CTA1_01-1</th>\n",
       "      <th>02i5jCrfQK,Section CTA1_01-2</th>\n",
       "      <th>02i5jCrfQK,Section CTA1_01-3</th>\n",
       "      <th>02i5jCrfQK,Section CTA1_02-1</th>\n",
       "      <th>02i5jCrfQK,Section CTA1_02-2</th>\n",
       "      <th>02i5jCrfQK,Section CTA1_02-3</th>\n",
       "      <th>02i5jCrfQK,Section CTA1_02-4</th>\n",
       "      <th>02i5jCrfQK,Section CTA1_04-1</th>\n",
       "      <th>02i5jCrfQK,Section CTA1_04-2</th>\n",
       "      <th>02i5jCrfQK,Section CTA1_06-1</th>\n",
       "      <th>...</th>\n",
       "      <th>zfYl0YECr0,Section ES_01-7</th>\n",
       "      <th>zfYl0YECr0,Section ES_01-8</th>\n",
       "      <th>zfYl0YECr0,Section ES_02-1</th>\n",
       "      <th>zfYl0YECr0,Section ES_02-2</th>\n",
       "      <th>zfYl0YECr0,Section ES_02-3</th>\n",
       "      <th>zfYl0YECr0,Section ES_02-4</th>\n",
       "      <th>zfYl0YECr0,Section ES_02-5</th>\n",
       "      <th>zfYl0YECr0,Section ES_02-6</th>\n",
       "      <th>zfYl0YECr0,Section ES_02-7</th>\n",
       "      <th>zfYl0YECr0,Section ES_02-8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 4760 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   02i5jCrfQK,Section CTA1_01-1  02i5jCrfQK,Section CTA1_01-2  \\\n",
       "0                             0                             0   \n",
       "1                             0                             0   \n",
       "2                             0                             0   \n",
       "3                             0                             0   \n",
       "4                             0                             0   \n",
       "\n",
       "   02i5jCrfQK,Section CTA1_01-3  02i5jCrfQK,Section CTA1_02-1  \\\n",
       "0                             0                             0   \n",
       "1                             0                             0   \n",
       "2                             0                             0   \n",
       "3                             0                             0   \n",
       "4                             0                             0   \n",
       "\n",
       "   02i5jCrfQK,Section CTA1_02-2  02i5jCrfQK,Section CTA1_02-3  \\\n",
       "0                             0                             0   \n",
       "1                             0                             0   \n",
       "2                             0                             0   \n",
       "3                             0                             0   \n",
       "4                             0                             0   \n",
       "\n",
       "   02i5jCrfQK,Section CTA1_02-4  02i5jCrfQK,Section CTA1_04-1  \\\n",
       "0                             0                             0   \n",
       "1                             0                             0   \n",
       "2                             0                             0   \n",
       "3                             0                             0   \n",
       "4                             0                             0   \n",
       "\n",
       "   02i5jCrfQK,Section CTA1_04-2  02i5jCrfQK,Section CTA1_06-1  ...  \\\n",
       "0                             0                             0  ...   \n",
       "1                             0                             0  ...   \n",
       "2                             0                             0  ...   \n",
       "3                             0                             0  ...   \n",
       "4                             0                             0  ...   \n",
       "\n",
       "   zfYl0YECr0,Section ES_01-7  zfYl0YECr0,Section ES_01-8  \\\n",
       "0                           0                           0   \n",
       "1                           0                           0   \n",
       "2                           0                           0   \n",
       "3                           0                           0   \n",
       "4                           0                           0   \n",
       "\n",
       "   zfYl0YECr0,Section ES_02-1  zfYl0YECr0,Section ES_02-2  \\\n",
       "0                           0                           0   \n",
       "1                           0                           0   \n",
       "2                           0                           0   \n",
       "3                           0                           0   \n",
       "4                           0                           0   \n",
       "\n",
       "   zfYl0YECr0,Section ES_02-3  zfYl0YECr0,Section ES_02-4  \\\n",
       "0                           0                           0   \n",
       "1                           0                           0   \n",
       "2                           0                           0   \n",
       "3                           0                           0   \n",
       "4                           0                           0   \n",
       "\n",
       "   zfYl0YECr0,Section ES_02-5  zfYl0YECr0,Section ES_02-6  \\\n",
       "0                           0                           0   \n",
       "1                           0                           0   \n",
       "2                           0                           0   \n",
       "3                           0                           0   \n",
       "4                           0                           0   \n",
       "\n",
       "   zfYl0YECr0,Section ES_02-7  zfYl0YECr0,Section ES_02-8  \n",
       "0                           0                           0  \n",
       "1                           0                           0  \n",
       "2                           0                           0  \n",
       "3                           0                           0  \n",
       "4                           0                           0  \n",
       "\n",
       "[5 rows x 4760 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Student Name and Problem Section\n",
    "cols = [\"Anon Student Id\", \"Problem Section\"]\n",
    "df[joint_name(cols)] = df.apply(join_columns, axis=1, args=(cols,))\n",
    "sid_ps_features = one_dim_split(df, joint_name(cols))\n",
    "sid_ps_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column Anon Student Id,Problem Name has 16236 values, so we will add as many columns to this dataframe.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>02i5jCrfQK,1PTB02</th>\n",
       "      <th>02i5jCrfQK,1PTB03</th>\n",
       "      <th>02i5jCrfQK,1PTB05</th>\n",
       "      <th>02i5jCrfQK,1PTB07</th>\n",
       "      <th>02i5jCrfQK,1PTB08</th>\n",
       "      <th>02i5jCrfQK,1PTB09</th>\n",
       "      <th>02i5jCrfQK,1PTFB10</th>\n",
       "      <th>02i5jCrfQK,1PTFB11</th>\n",
       "      <th>02i5jCrfQK,1PTFB13</th>\n",
       "      <th>02i5jCrfQK,1PTFB14</th>\n",
       "      <th>...</th>\n",
       "      <th>zfYl0YECr0,PERCENT01C</th>\n",
       "      <th>zfYl0YECr0,PROP03</th>\n",
       "      <th>zfYl0YECr0,PROP04</th>\n",
       "      <th>zfYl0YECr0,PROP05</th>\n",
       "      <th>zfYl0YECr0,PROP07</th>\n",
       "      <th>zfYl0YECr0,PROP08</th>\n",
       "      <th>zfYl0YECr0,PROP12</th>\n",
       "      <th>zfYl0YECr0,REAL13C</th>\n",
       "      <th>zfYl0YECr0,REAL19B</th>\n",
       "      <th>zfYl0YECr0,REAL28</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 16236 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   02i5jCrfQK,1PTB02  02i5jCrfQK,1PTB03  02i5jCrfQK,1PTB05  02i5jCrfQK,1PTB07  \\\n",
       "0                  0                  0                  0                  0   \n",
       "1                  0                  0                  0                  0   \n",
       "2                  0                  0                  0                  0   \n",
       "3                  0                  0                  0                  0   \n",
       "4                  0                  0                  0                  0   \n",
       "\n",
       "   02i5jCrfQK,1PTB08  02i5jCrfQK,1PTB09  02i5jCrfQK,1PTFB10  \\\n",
       "0                  0                  0                   0   \n",
       "1                  0                  0                   0   \n",
       "2                  0                  0                   0   \n",
       "3                  0                  0                   0   \n",
       "4                  0                  0                   0   \n",
       "\n",
       "   02i5jCrfQK,1PTFB11  02i5jCrfQK,1PTFB13  02i5jCrfQK,1PTFB14  ...  \\\n",
       "0                   0                   0                   0  ...   \n",
       "1                   0                   0                   0  ...   \n",
       "2                   0                   0                   0  ...   \n",
       "3                   0                   0                   0  ...   \n",
       "4                   0                   0                   0  ...   \n",
       "\n",
       "   zfYl0YECr0,PERCENT01C  zfYl0YECr0,PROP03  zfYl0YECr0,PROP04  \\\n",
       "0                      0                  0                  0   \n",
       "1                      0                  0                  0   \n",
       "2                      0                  0                  0   \n",
       "3                      0                  0                  0   \n",
       "4                      0                  0                  0   \n",
       "\n",
       "   zfYl0YECr0,PROP05  zfYl0YECr0,PROP07  zfYl0YECr0,PROP08  zfYl0YECr0,PROP12  \\\n",
       "0                  0                  0                  0                  0   \n",
       "1                  0                  0                  0                  0   \n",
       "2                  0                  0                  0                  0   \n",
       "3                  0                  0                  0                  0   \n",
       "4                  0                  0                  0                  0   \n",
       "\n",
       "   zfYl0YECr0,REAL13C  zfYl0YECr0,REAL19B  zfYl0YECr0,REAL28  \n",
       "0                   0                   0                  0  \n",
       "1                   0                   0                  0  \n",
       "2                   0                   0                  0  \n",
       "3                   0                   0                  0  \n",
       "4                   0                   0                  0  \n",
       "\n",
       "[5 rows x 16236 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Student Name and Problem Name\n",
    "cols = [\"Anon Student Id\", \"Problem Name\"]\n",
    "df[joint_name(cols)] = df.apply(join_columns, axis=1, args=(cols,))\n",
    "sid_pn_features = one_dim_split(df, joint_name(cols))\n",
    "sid_pn_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column Anon Student Id,Step Name has 91699 values, so we will add as many columns to this dataframe.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>02i5jCrfQK,(-1+8.54400375)/2 = x</th>\n",
       "      <th>02i5jCrfQK,(-1+sqrt(1-(-72)))/2 = x</th>\n",
       "      <th>02i5jCrfQK,(-1+sqrt(1-4*1*-18))/2 = x</th>\n",
       "      <th>02i5jCrfQK,(-1+sqrt(73))/2 = x</th>\n",
       "      <th>02i5jCrfQK,(-1-8.54400375)/2 = x</th>\n",
       "      <th>02i5jCrfQK,(-1-sqrt(1-(-72)))/2 = x</th>\n",
       "      <th>02i5jCrfQK,(-1-sqrt(1-4*1*-18))/2 = x</th>\n",
       "      <th>02i5jCrfQK,(-1-sqrt(73))/2 = x</th>\n",
       "      <th>02i5jCrfQK,(-1000+sqrt(1000000-4*-16*-1950))/-32 = x</th>\n",
       "      <th>02i5jCrfQK,(-1000+sqrt(1000000-4*-16*50))/-32 = x</th>\n",
       "      <th>...</th>\n",
       "      <th>zfYl0YECr0,y/-3 = 1</th>\n",
       "      <th>zfYl0YECr0,y/-3-9 = -8</th>\n",
       "      <th>zfYl0YECr0,y/-3-9+9 = -8+9</th>\n",
       "      <th>zfYl0YECr0,y/-3-9+9 = 1</th>\n",
       "      <th>zfYl0YECr0,y/9 = 1</th>\n",
       "      <th>zfYl0YECr0,y/9-5 = -4</th>\n",
       "      <th>zfYl0YECr0,y/9-5+5 = -4+5</th>\n",
       "      <th>zfYl0YECr0,yMax</th>\n",
       "      <th>zfYl0YECr0,yMin</th>\n",
       "      <th>zfYl0YECr0,yScale</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 91699 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   02i5jCrfQK,(-1+8.54400375)/2 = x  02i5jCrfQK,(-1+sqrt(1-(-72)))/2 = x  \\\n",
       "0                                 0                                    0   \n",
       "1                                 0                                    0   \n",
       "2                                 0                                    0   \n",
       "3                                 0                                    0   \n",
       "4                                 0                                    0   \n",
       "\n",
       "   02i5jCrfQK,(-1+sqrt(1-4*1*-18))/2 = x  02i5jCrfQK,(-1+sqrt(73))/2 = x  \\\n",
       "0                                      0                               0   \n",
       "1                                      0                               0   \n",
       "2                                      0                               0   \n",
       "3                                      0                               0   \n",
       "4                                      0                               0   \n",
       "\n",
       "   02i5jCrfQK,(-1-8.54400375)/2 = x  02i5jCrfQK,(-1-sqrt(1-(-72)))/2 = x  \\\n",
       "0                                 0                                    0   \n",
       "1                                 0                                    0   \n",
       "2                                 0                                    0   \n",
       "3                                 0                                    0   \n",
       "4                                 0                                    0   \n",
       "\n",
       "   02i5jCrfQK,(-1-sqrt(1-4*1*-18))/2 = x  02i5jCrfQK,(-1-sqrt(73))/2 = x  \\\n",
       "0                                      0                               0   \n",
       "1                                      0                               0   \n",
       "2                                      0                               0   \n",
       "3                                      0                               0   \n",
       "4                                      0                               0   \n",
       "\n",
       "   02i5jCrfQK,(-1000+sqrt(1000000-4*-16*-1950))/-32 = x  \\\n",
       "0                                                  0      \n",
       "1                                                  0      \n",
       "2                                                  0      \n",
       "3                                                  0      \n",
       "4                                                  0      \n",
       "\n",
       "   02i5jCrfQK,(-1000+sqrt(1000000-4*-16*50))/-32 = x  ...  \\\n",
       "0                                                  0  ...   \n",
       "1                                                  0  ...   \n",
       "2                                                  0  ...   \n",
       "3                                                  0  ...   \n",
       "4                                                  0  ...   \n",
       "\n",
       "   zfYl0YECr0,y/-3 = 1  zfYl0YECr0,y/-3-9 = -8  zfYl0YECr0,y/-3-9+9 = -8+9  \\\n",
       "0                    0                       0                           0   \n",
       "1                    0                       0                           0   \n",
       "2                    0                       0                           0   \n",
       "3                    0                       0                           0   \n",
       "4                    0                       0                           0   \n",
       "\n",
       "   zfYl0YECr0,y/-3-9+9 = 1  zfYl0YECr0,y/9 = 1  zfYl0YECr0,y/9-5 = -4  \\\n",
       "0                        0                   0                      0   \n",
       "1                        0                   0                      0   \n",
       "2                        0                   0                      0   \n",
       "3                        0                   0                      0   \n",
       "4                        0                   0                      0   \n",
       "\n",
       "   zfYl0YECr0,y/9-5+5 = -4+5  zfYl0YECr0,yMax  zfYl0YECr0,yMin  \\\n",
       "0                          0                0                0   \n",
       "1                          0                0                0   \n",
       "2                          0                0                0   \n",
       "3                          0                0                0   \n",
       "4                          0                0                0   \n",
       "\n",
       "   zfYl0YECr0,yScale  \n",
       "0                  0  \n",
       "1                  0  \n",
       "2                  0  \n",
       "3                  0  \n",
       "4                  0  \n",
       "\n",
       "[5 rows x 91699 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Student Name and Step Name\n",
    "cols = [\"Anon Student Id\", \"Step Name\"]\n",
    "df[joint_name(cols)] = df.apply(join_columns, axis=1, args=(cols,))\n",
    "sid_sn_features = one_dim_split(df, joint_name(cols))\n",
    "sid_sn_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column Anon Student Id,Problem Unit,Problem Section,Problem Name,Step Name has 218050 values, so we will add as many columns to this dataframe.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R1C1</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R1C2</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R2C1</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R2C2</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R3C1</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R3C2</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R4C1</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R4C2</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R5C1</th>\n",
       "      <th>02i5jCrfQK,Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R5C2</th>\n",
       "      <th>...</th>\n",
       "      <th>zfYl0YECr0,Unit ES_02,Section ES_02-7,EG41,y/-3-9+9 = -8+9</th>\n",
       "      <th>zfYl0YECr0,Unit ES_02,Section ES_02-7,EG41,y/-3-9+9 = 1</th>\n",
       "      <th>zfYl0YECr0,Unit ES_02,Section ES_02-8,EG41,(y/9)/(1/9) = 1/(1/9)</th>\n",
       "      <th>zfYl0YECr0,Unit ES_02,Section ES_02-8,EG41,5 = x/10</th>\n",
       "      <th>zfYl0YECr0,Unit ES_02,Section ES_02-8,EG41,5/(1/10) = (x/10)/(1/10)</th>\n",
       "      <th>zfYl0YECr0,Unit ES_02,Section ES_02-8,EG41,8 = x/10+3</th>\n",
       "      <th>zfYl0YECr0,Unit ES_02,Section ES_02-8,EG41,8-3 = x/10+3-3</th>\n",
       "      <th>zfYl0YECr0,Unit ES_02,Section ES_02-8,EG41,y/9 = 1</th>\n",
       "      <th>zfYl0YECr0,Unit ES_02,Section ES_02-8,EG41,y/9-5 = -4</th>\n",
       "      <th>zfYl0YECr0,Unit ES_02,Section ES_02-8,EG41,y/9-5+5 = -4+5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 218050 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   02i5jCrfQK,Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R1C1  \\\n",
       "0                                                  0           \n",
       "1                                                  0           \n",
       "2                                                  0           \n",
       "3                                                  0           \n",
       "4                                                  0           \n",
       "\n",
       "   02i5jCrfQK,Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R1C2  \\\n",
       "0                                                  0           \n",
       "1                                                  0           \n",
       "2                                                  0           \n",
       "3                                                  0           \n",
       "4                                                  0           \n",
       "\n",
       "   02i5jCrfQK,Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R2C1  \\\n",
       "0                                                  0           \n",
       "1                                                  0           \n",
       "2                                                  0           \n",
       "3                                                  0           \n",
       "4                                                  0           \n",
       "\n",
       "   02i5jCrfQK,Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R2C2  \\\n",
       "0                                                  0           \n",
       "1                                                  0           \n",
       "2                                                  0           \n",
       "3                                                  0           \n",
       "4                                                  0           \n",
       "\n",
       "   02i5jCrfQK,Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R3C1  \\\n",
       "0                                                  0           \n",
       "1                                                  0           \n",
       "2                                                  0           \n",
       "3                                                  0           \n",
       "4                                                  0           \n",
       "\n",
       "   02i5jCrfQK,Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R3C2  \\\n",
       "0                                                  0           \n",
       "1                                                  0           \n",
       "2                                                  0           \n",
       "3                                                  0           \n",
       "4                                                  0           \n",
       "\n",
       "   02i5jCrfQK,Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R4C1  \\\n",
       "0                                                  0           \n",
       "1                                                  0           \n",
       "2                                                  0           \n",
       "3                                                  0           \n",
       "4                                                  0           \n",
       "\n",
       "   02i5jCrfQK,Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R4C2  \\\n",
       "0                                                  0           \n",
       "1                                                  0           \n",
       "2                                                  0           \n",
       "3                                                  0           \n",
       "4                                                  0           \n",
       "\n",
       "   02i5jCrfQK,Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R5C1  \\\n",
       "0                                                  0           \n",
       "1                                                  0           \n",
       "2                                                  0           \n",
       "3                                                  0           \n",
       "4                                                  0           \n",
       "\n",
       "   02i5jCrfQK,Unit CTA1_01,Section CTA1_01-1,LDEMO_WKST,R5C2  ...  \\\n",
       "0                                                  0          ...   \n",
       "1                                                  0          ...   \n",
       "2                                                  0          ...   \n",
       "3                                                  0          ...   \n",
       "4                                                  0          ...   \n",
       "\n",
       "   zfYl0YECr0,Unit ES_02,Section ES_02-7,EG41,y/-3-9+9 = -8+9  \\\n",
       "0                                                  0            \n",
       "1                                                  0            \n",
       "2                                                  0            \n",
       "3                                                  0            \n",
       "4                                                  0            \n",
       "\n",
       "   zfYl0YECr0,Unit ES_02,Section ES_02-7,EG41,y/-3-9+9 = 1  \\\n",
       "0                                                  0         \n",
       "1                                                  0         \n",
       "2                                                  0         \n",
       "3                                                  0         \n",
       "4                                                  0         \n",
       "\n",
       "   zfYl0YECr0,Unit ES_02,Section ES_02-8,EG41,(y/9)/(1/9) = 1/(1/9)  \\\n",
       "0                                                  0                  \n",
       "1                                                  0                  \n",
       "2                                                  0                  \n",
       "3                                                  0                  \n",
       "4                                                  0                  \n",
       "\n",
       "   zfYl0YECr0,Unit ES_02,Section ES_02-8,EG41,5 = x/10  \\\n",
       "0                                                  0     \n",
       "1                                                  0     \n",
       "2                                                  0     \n",
       "3                                                  0     \n",
       "4                                                  0     \n",
       "\n",
       "   zfYl0YECr0,Unit ES_02,Section ES_02-8,EG41,5/(1/10) = (x/10)/(1/10)  \\\n",
       "0                                                  0                     \n",
       "1                                                  0                     \n",
       "2                                                  0                     \n",
       "3                                                  0                     \n",
       "4                                                  0                     \n",
       "\n",
       "   zfYl0YECr0,Unit ES_02,Section ES_02-8,EG41,8 = x/10+3  \\\n",
       "0                                                  0       \n",
       "1                                                  0       \n",
       "2                                                  0       \n",
       "3                                                  0       \n",
       "4                                                  0       \n",
       "\n",
       "   zfYl0YECr0,Unit ES_02,Section ES_02-8,EG41,8-3 = x/10+3-3  \\\n",
       "0                                                  0           \n",
       "1                                                  0           \n",
       "2                                                  0           \n",
       "3                                                  0           \n",
       "4                                                  0           \n",
       "\n",
       "   zfYl0YECr0,Unit ES_02,Section ES_02-8,EG41,y/9 = 1  \\\n",
       "0                                                  0    \n",
       "1                                                  0    \n",
       "2                                                  0    \n",
       "3                                                  0    \n",
       "4                                                  0    \n",
       "\n",
       "   zfYl0YECr0,Unit ES_02,Section ES_02-8,EG41,y/9-5 = -4  \\\n",
       "0                                                  0       \n",
       "1                                                  0       \n",
       "2                                                  0       \n",
       "3                                                  0       \n",
       "4                                                  0       \n",
       "\n",
       "   zfYl0YECr0,Unit ES_02,Section ES_02-8,EG41,y/9-5+5 = -4+5  \n",
       "0                                                  0          \n",
       "1                                                  0          \n",
       "2                                                  0          \n",
       "3                                                  0          \n",
       "4                                                  0          \n",
       "\n",
       "[5 rows x 218050 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Student Name and Problem Unit and Problem Section and Problem Name and Step Name\n",
    "cols = [\"Anon Student Id\", \"Problem Unit\", \"Problem Section\", \"Problem Name\", \"Step Name\"]\n",
    "df[joint_name(cols)] = df.apply(join_columns, axis=1, args=(cols,))\n",
    "sid_pu_ps_pn_sn_features = one_dim_split(df, joint_name(cols))\n",
    "sid_pu_ps_pn_sn_features.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Knowledge Components"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_kc_dummies(row, headers, col):\n",
    "    tokens = row[col].split(\"~~\")\n",
    "    opps = np.asarray([int(s) if s.lower() != \"nan\" else 0 for s in row[\"Opportunity(Default)\"].split(\"~~\")])\n",
    "    opps = np.log(opps + 1)\n",
    "    sr = pd.Series(np.zeros(len(headers)), index = headers)\n",
    "    sr[tokens] = opps\n",
    "    return sr.to_sparse(fill_value=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KC(Default) has 349 compound values.\n",
      "There are 111 atomic KCs, so we will be adding as many columns to this dataframe\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>[SkillRule: invert-mult; {ivm}]</th>\n",
       "      <th>Positive Constants, SIF</th>\n",
       "      <th>Write expression, ratio</th>\n",
       "      <th>Shading GLF equation with positive slope</th>\n",
       "      <th>Edit Algebraic a</th>\n",
       "      <th>Find X, positive slope</th>\n",
       "      <th>[SkillRule: Eliminate Parens; {CLT nested; CLT nested, parens; Distribute Mult right; Distribute Mult left; (+/-x +/-a)/b=c, mult; (+/-x +/-a)*b=c, div; [var expr]/[const expr] = [const expr], multiply; Distribute Division left; Distribute Division right; Distribute both mult left; Distribute both mult right; Distribute both divide left; Distribute both divide right; Distribute subex}]</th>\n",
       "      <th>Shading SIF equation with negative slope</th>\n",
       "      <th>Entering a computed linear value</th>\n",
       "      <th>Placing coordinate point</th>\n",
       "      <th>...</th>\n",
       "      <th>[SkillRule: ax+b=c, negative; ax+b=c, negative]</th>\n",
       "      <th>Choose Graphical a</th>\n",
       "      <th>[SkillRule: Consolidate vars, no coeff; CLT]</th>\n",
       "      <th>Including the line when shading</th>\n",
       "      <th>Using small numbers</th>\n",
       "      <th>Convert unit, multiplier</th>\n",
       "      <th>[SkillRule: Select Multiply; {MT; MT no fraction coeff}]</th>\n",
       "      <th>Find Y, negative slope</th>\n",
       "      <th>Entering x-intercept, SIF</th>\n",
       "      <th>Using difficult numbers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.693147</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.693147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 111 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   [SkillRule: invert-mult; {ivm}]  Positive Constants, SIF  \\\n",
       "0                              0.0                      0.0   \n",
       "1                              0.0                      0.0   \n",
       "2                              0.0                      0.0   \n",
       "3                              0.0                      0.0   \n",
       "4                              0.0                      0.0   \n",
       "\n",
       "   Write expression, ratio  Shading GLF equation with positive slope  \\\n",
       "0                      0.0                                       0.0   \n",
       "1                      0.0                                       0.0   \n",
       "2                      0.0                                       0.0   \n",
       "3                      0.0                                       0.0   \n",
       "4                      0.0                                       0.0   \n",
       "\n",
       "   Edit Algebraic a  Find X, positive slope  \\\n",
       "0               0.0                     0.0   \n",
       "1               0.0                     0.0   \n",
       "2               0.0                     0.0   \n",
       "3               0.0                     0.0   \n",
       "4               0.0                     0.0   \n",
       "\n",
       "   [SkillRule: Eliminate Parens; {CLT nested; CLT nested, parens; Distribute Mult right; Distribute Mult left; (+/-x +/-a)/b=c, mult; (+/-x +/-a)*b=c, div; [var expr]/[const expr] = [const expr], multiply; Distribute Division left; Distribute Division right; Distribute both mult left; Distribute both mult right; Distribute both divide left; Distribute both divide right; Distribute subex}]  \\\n",
       "0                                                0.0                                                                                                                                                                                                                                                                                                                                                      \n",
       "1                                                0.0                                                                                                                                                                                                                                                                                                                                                      \n",
       "2                                                0.0                                                                                                                                                                                                                                                                                                                                                      \n",
       "3                                                0.0                                                                                                                                                                                                                                                                                                                                                      \n",
       "4                                                0.0                                                                                                                                                                                                                                                                                                                                                      \n",
       "\n",
       "   Shading SIF equation with negative slope  Entering a computed linear value  \\\n",
       "0                                       0.0                               0.0   \n",
       "1                                       0.0                               0.0   \n",
       "2                                       0.0                               0.0   \n",
       "3                                       0.0                               0.0   \n",
       "4                                       0.0                               0.0   \n",
       "\n",
       "   Placing coordinate point  ...  \\\n",
       "0                       0.0  ...   \n",
       "1                       0.0  ...   \n",
       "2                       0.0  ...   \n",
       "3                       0.0  ...   \n",
       "4                       0.0  ...   \n",
       "\n",
       "   [SkillRule: ax+b=c, negative; ax+b=c, negative]  Choose Graphical a  \\\n",
       "0                                              0.0                 0.0   \n",
       "1                                              0.0                 0.0   \n",
       "2                                              0.0                 0.0   \n",
       "3                                              0.0                 0.0   \n",
       "4                                              0.0                 0.0   \n",
       "\n",
       "   [SkillRule: Consolidate vars, no coeff; CLT]  \\\n",
       "0                                           0.0   \n",
       "1                                           0.0   \n",
       "2                                           0.0   \n",
       "3                                           0.0   \n",
       "4                                           0.0   \n",
       "\n",
       "   Including the line when shading  Using small numbers  \\\n",
       "0                              0.0             0.000000   \n",
       "1                              0.0             0.000000   \n",
       "2                              0.0             0.000000   \n",
       "3                              0.0             0.693147   \n",
       "4                              0.0             0.000000   \n",
       "\n",
       "   Convert unit, multiplier  \\\n",
       "0                       0.0   \n",
       "1                       0.0   \n",
       "2                       0.0   \n",
       "3                       0.0   \n",
       "4                       0.0   \n",
       "\n",
       "   [SkillRule: Select Multiply; {MT; MT no fraction coeff}]  \\\n",
       "0                                                0.0          \n",
       "1                                                0.0          \n",
       "2                                                0.0          \n",
       "3                                                0.0          \n",
       "4                                                0.0          \n",
       "\n",
       "   Find Y, negative slope  Entering x-intercept, SIF  Using difficult numbers  \n",
       "0                     0.0                        0.0                 0.000000  \n",
       "1                     0.0                        0.0                 0.000000  \n",
       "2                     0.0                        0.0                 0.000000  \n",
       "3                     0.0                        0.0                 0.693147  \n",
       "4                     0.0                        0.0                 0.000000  \n",
       "\n",
       "[5 rows x 111 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compound_KCs = set(df[\"KC(Default)\"])\n",
    "print(\"KC(Default) has %d compound values.\" % len(compound_KCs))\n",
    "KCs = set()\n",
    "for s in compound_KCs:\n",
    "    KCs = KCs.union(set(s.split(\"~~\")))\n",
    "\n",
    "print(\"There are %d atomic KCs, so we will be adding as many columns to this dataframe\" % len(KCs))\n",
    "\n",
    "# Cast and split columns\n",
    "kc_features = df.apply(get_kc_dummies, axis=1, result_type=\"expand\", args=(KCs, \"KC(Default)\"))\n",
    "kc_features.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Numerical Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Problem View\n",
    "def norm_problem_view(row):\n",
    "    return math.log(int(row[\"Problem View\"]) + 1)\n",
    "pv_features = df.apply(norm_problem_view, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Opportunity\n",
    "def norm_opportunity(row):\n",
    "    opps = np.asarray([int(s) if s.lower() != \"nan\" else 0 for s in row[\"Opportunity(Default)\"].split(\"~~\")])\n",
    "    return math.log(np.min(opps) + 1)\n",
    "opp_features = df.apply(norm_opportunity, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "numerical_features = pd.concat((pv_features, opp_features), axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Aggregation and Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import sparse\n",
    "features = (pu_features, ps_features\n",
    "    , sn_features, sid_features, kc_features\n",
    "    , numerical_features\n",
    "    , sid_pu_features, pu_ps_features\n",
    "    , ps_pn_features, pn_sn_features\n",
    "    , sid_pu_ps_features, pu_ps_pn_features\n",
    "    , ps_pn_sn_features, sid_pu_ps_pn_features\n",
    "    , pu_ps_pn_sn_features\n",
    "    , sid_ps_features, sid_pn_features\n",
    "    , sid_sn_features, sid_pu_ps_pn_sn_features\n",
    ")\n",
    "aggdf = sparse.hstack([f.to_sparse(fill_value=0).to_coo().tocsr() for f in features], format=\"csr\", dtype=float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Table X: (232744, 658691)\n",
      "Table Y: (232744,)\n"
     ]
    }
   ],
   "source": [
    "# Split training set and testing set\n",
    "X = aggdf[0:len(traindf)]\n",
    "Y = traindf[\"Correct First Attempt\"]\n",
    "print(\"Table X:\", X.shape)\n",
    "print(\"Table Y:\", Y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Table X_: (1140, 658691)\n",
      "Table Y_: (1140,)\n"
     ]
    }
   ],
   "source": [
    "X_ = aggdf[len(traindf):]\n",
    "Y_ = testdf[\"Correct First Attempt\"]\n",
    "print(\"Table X_:\", X_.shape)\n",
    "print(\"Table Y_:\", Y_.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Machine Learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import linear_model\n",
    "lr_model = linear_model.LogisticRegression(solver=\"liblinear\", n_jobs=-1, max_iter=1000) # Enable all CPUs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:1544: UserWarning: 'n_jobs' > 1 does not have any effect when 'solver' is set to 'liblinear'. Got 'n_jobs' = 8.\n",
      "  \" = {}.\".format(effective_n_jobs(self.n_jobs)))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=1000,\n",
       "                   multi_class='warn', n_jobs=-1, penalty='l2',\n",
       "                   random_state=None, solver='liblinear', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_model.fit(X, Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import ensemble\n",
    "est_count = 100\n",
    "rf_model = ensemble.RandomForestClassifier(n_estimators = est_count, criterion=\"entropy\", max_depth=7, n_jobs=-1) # Enable all CPUs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='entropy',\n",
       "                       max_depth=7, max_features='auto', max_leaf_nodes=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, n_estimators=100,\n",
       "                       n_jobs=-1, oob_score=False, random_state=None, verbose=0,\n",
       "                       warm_start=False)"
      ]
     },
     "execution_count": 323,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_model.fit(X, Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AdaBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import ensemble\n",
    "from sklearn import linear_model\n",
    "ab_model = ensemble.AdaBoostClassifier(base_estimator=linear_model.LogisticRegression(solver=\"saga\", n_jobs=-1, max_iter=1000))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AdaBoostClassifier(algorithm='SAMME.R',\n",
       "                   base_estimator=LogisticRegression(C=1.0, class_weight=None,\n",
       "                                                     dual=False,\n",
       "                                                     fit_intercept=True,\n",
       "                                                     intercept_scaling=1,\n",
       "                                                     l1_ratio=None,\n",
       "                                                     max_iter=1000,\n",
       "                                                     multi_class='warn',\n",
       "                                                     n_jobs=-1, penalty='l2',\n",
       "                                                     random_state=None,\n",
       "                                                     solver='saga', tol=0.0001,\n",
       "                                                     verbose=0,\n",
       "                                                     warm_start=False),\n",
       "                   learning_rate=1.0, n_estimators=50, random_state=None)"
      ]
     },
     "execution_count": 341,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ab_model.fit(X, Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Error Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Root Mean Squared Error\n",
    "# Here, we consider using numpy as a powerful\n",
    "# utility to solve the RMSE\n",
    "def RMSE(P, Y):\n",
    "    P = P[~np.isnan(Y)]\n",
    "    Y = Y[~np.isnan(Y)]\n",
    "    return np.sqrt(np.sum(np.square(P - Y)) / len(Y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Error: 0.22914622805505377\n",
      "Test Error: 0.3443998310806608\n"
     ]
    }
   ],
   "source": [
    "# Logistic Classification\n",
    "\n",
    "P = lr_model.predict_proba(X)[:, 1]\n",
    "P_ = lr_model.predict_proba(X_)[:, 1]\n",
    "\n",
    "print(\"Train Error:\", RMSE(P, Y))\n",
    "print(\"Test Error:\", RMSE(P_, Y_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate submission\n",
    "RES = P_[np.isnan(Y_)]\n",
    "testdf.loc[np.isnan(testdf[\"Correct First Attempt\"]), \"Correct First Attempt\"] = RES\n",
    "testdf.to_csv(\"data/predict_result.csv\", sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save\n",
    "pd.Series(P).to_csv(\"data/sparse_train.csv\", sep='\\t', header=[\"sparse_res\"])\n",
    "pd.Series(P_).to_csv(\"data/sparse_test.csv\", sep='\\t', header=[\"sparse_res\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Error: 0.4114884197701772\n",
      "Test Error: 0.3910843073433951\n"
     ]
    }
   ],
   "source": [
    "# Random Forest\n",
    "\n",
    "P = rf_model.predict_proba(X)[:, 1]\n",
    "P_ = rf_model.predict_proba(X_)[:, 1]\n",
    "\n",
    "print(\"Train Error:\", RMSE(P, Y))\n",
    "print(\"Test Error:\", RMSE(P_, Y_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Error: 0.49575808621765616\n",
      "Test Error: 0.4954684100106574\n"
     ]
    }
   ],
   "source": [
    "# AdaBoost\n",
    "\n",
    "P = ab_model.predict_proba(X)[:, 1]\n",
    "P_ = ab_model.predict_proba(X_)[:, 1]\n",
    "\n",
    "print(\"Train Error:\", RMSE(P, Y))\n",
    "print(\"Test Error:\", RMSE(P_, Y_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}