removing_duplicates_1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Remove duplicates:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=pd.read_pickle('pickels/28k_apparel_data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>asin</th>\n",
       "      <th>brand</th>\n",
       "      <th>color</th>\n",
       "      <th>medium_image_url</th>\n",
       "      <th>product_type_name</th>\n",
       "      <th>title</th>\n",
       "      <th>formatted_price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>B004GSI2OS</td>\n",
       "      <td>FeatherLite</td>\n",
       "      <td>Onyx Black/ Stone</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "      <td>SHIRT</td>\n",
       "      <td>Featherlite Ladies' Long Sleeve Stain Resistan...</td>\n",
       "      <td>$26.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>B012YX2ZPI</td>\n",
       "      <td>HX-Kingdom Fashion T-shirts</td>\n",
       "      <td>White</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "      <td>SHIRT</td>\n",
       "      <td>Women's Unique 100% Cotton T - Special Olympic...</td>\n",
       "      <td>$9.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>B001LOUGE4</td>\n",
       "      <td>Fitness Etc.</td>\n",
       "      <td>Black</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "      <td>SHIRT</td>\n",
       "      <td>Ladies Cotton Tank 2x1 Ribbed Tank Top</td>\n",
       "      <td>$11.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>B003BSRPB0</td>\n",
       "      <td>FeatherLite</td>\n",
       "      <td>White</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "      <td>SHIRT</td>\n",
       "      <td>FeatherLite Ladies' Moisture Free Mesh Sport S...</td>\n",
       "      <td>$20.54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>B014ICEDNA</td>\n",
       "      <td>FNC7C</td>\n",
       "      <td>Purple</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "      <td>SHIRT</td>\n",
       "      <td>Supernatural Chibis Sam Dean And Castiel Short...</td>\n",
       "      <td>$7.50</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          asin                        brand              color  \\\n",
       "4   B004GSI2OS                  FeatherLite  Onyx Black/ Stone   \n",
       "6   B012YX2ZPI  HX-Kingdom Fashion T-shirts              White   \n",
       "11  B001LOUGE4                 Fitness Etc.              Black   \n",
       "15  B003BSRPB0                  FeatherLite              White   \n",
       "21  B014ICEDNA                        FNC7C             Purple   \n",
       "\n",
       "                                     medium_image_url product_type_name  \\\n",
       "4   https://images-na.ssl-images-amazon.com/images...             SHIRT   \n",
       "6   https://images-na.ssl-images-amazon.com/images...             SHIRT   \n",
       "11  https://images-na.ssl-images-amazon.com/images...             SHIRT   \n",
       "15  https://images-na.ssl-images-amazon.com/images...             SHIRT   \n",
       "21  https://images-na.ssl-images-amazon.com/images...             SHIRT   \n",
       "\n",
       "                                                title formatted_price  \n",
       "4   Featherlite Ladies' Long Sleeve Stain Resistan...          $26.26  \n",
       "6   Women's Unique 100% Cotton T - Special Olympic...           $9.99  \n",
       "11             Ladies Cotton Tank 2x1 Ribbed Tank Top          $11.99  \n",
       "15  FeatherLite Ladies' Moisture Free Mesh Sport S...          $20.54  \n",
       "21  Supernatural Chibis Sam Dean And Castiel Short...           $7.50  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "After removal of products with short decription:  27949\n"
     ]
    }
   ],
   "source": [
    "#Removing data with short titles as they do not give any detailed relevant description\n",
    "data_sorted=data[data['title'].apply(lambda x: len(x.split())>4)]\n",
    "print(\"After removal of products with short decription: \",data_sorted.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>asin</th>\n",
       "      <th>brand</th>\n",
       "      <th>color</th>\n",
       "      <th>medium_image_url</th>\n",
       "      <th>product_type_name</th>\n",
       "      <th>title</th>\n",
       "      <th>formatted_price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>61973</th>\n",
       "      <td>B06Y1KZ2WB</td>\n",
       "      <td>Éclair</td>\n",
       "      <td>Black/Pink</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "      <td>SHIRT</td>\n",
       "      <td>Éclair Women's Printed Thin Strap Blouse Black...</td>\n",
       "      <td>$24.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133820</th>\n",
       "      <td>B010RV33VE</td>\n",
       "      <td>xiaoming</td>\n",
       "      <td>Pink</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "      <td>SHIRT</td>\n",
       "      <td>xiaoming Womens Sleeveless Loose Long T-shirts...</td>\n",
       "      <td>$18.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81461</th>\n",
       "      <td>B01DDSDLNS</td>\n",
       "      <td>xiaoming</td>\n",
       "      <td>White</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "      <td>SHIRT</td>\n",
       "      <td>xiaoming Women's White Long Sleeve Single Brea...</td>\n",
       "      <td>$21.58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75995</th>\n",
       "      <td>B00X5LYO9Y</td>\n",
       "      <td>xiaoming</td>\n",
       "      <td>Red Anchors</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "      <td>SHIRT</td>\n",
       "      <td>xiaoming Stripes Tank Patch/Bear Sleeve Anchor...</td>\n",
       "      <td>$15.91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151570</th>\n",
       "      <td>B00WPJG35K</td>\n",
       "      <td>xiaoming</td>\n",
       "      <td>White</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "      <td>SHIRT</td>\n",
       "      <td>xiaoming Sleeve Sheer Loose Tassel Kimono Woma...</td>\n",
       "      <td>$14.32</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              asin     brand        color  \\\n",
       "61973   B06Y1KZ2WB    Éclair   Black/Pink   \n",
       "133820  B010RV33VE  xiaoming         Pink   \n",
       "81461   B01DDSDLNS  xiaoming        White   \n",
       "75995   B00X5LYO9Y  xiaoming  Red Anchors   \n",
       "151570  B00WPJG35K  xiaoming        White   \n",
       "\n",
       "                                         medium_image_url product_type_name  \\\n",
       "61973   https://images-na.ssl-images-amazon.com/images...             SHIRT   \n",
       "133820  https://images-na.ssl-images-amazon.com/images...             SHIRT   \n",
       "81461   https://images-na.ssl-images-amazon.com/images...             SHIRT   \n",
       "75995   https://images-na.ssl-images-amazon.com/images...             SHIRT   \n",
       "151570  https://images-na.ssl-images-amazon.com/images...             SHIRT   \n",
       "\n",
       "                                                    title formatted_price  \n",
       "61973   Éclair Women's Printed Thin Strap Blouse Black...          $24.99  \n",
       "133820  xiaoming Womens Sleeveless Loose Long T-shirts...          $18.19  \n",
       "81461   xiaoming Women's White Long Sleeve Single Brea...          $21.58  \n",
       "75995   xiaoming Stripes Tank Patch/Bear Sleeve Anchor...          $15.91  \n",
       "151570  xiaoming Sleeve Sheer Loose Tassel Kimono Woma...          $14.32  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sort the whole data in alphabetical order of title\n",
    "data_sorted.sort_values('title',inplace=True, ascending=False)\n",
    "data_sorted.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Some examples of dupliacte titles that differ only in the last few words."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Titles 1:\n",
    "16. woman's place is in the house and the senate shirts for Womens XXL White\n",
    "17. woman's place is in the house and the senate shirts for Womens M Grey\n",
    "\n",
    "Title 2:\n",
    "25. tokidoki The Queen of Diamonds Women's Shirt X-Large\n",
    "26. tokidoki The Queen of Diamonds Women's Shirt Small\n",
    "27. tokidoki The Queen of Diamonds Women's Shirt Large\n",
    "\n",
    "Title 3:\n",
    "61. psychedelic colorful Howling Galaxy Wolf T-shirt/Colorful Rainbow Animal Print Head Shirt for woman Neon Wolf t-shirt\n",
    "62. psychedelic colorful Howling Galaxy Wolf T-shirt/Colorful Rainbow Animal Print Head Shirt for woman Neon Wolf t-shirt\n",
    "63. psychedelic colorful Howling Galaxy Wolf T-shirt/Colorful Rainbow Animal Print Head Shirt for woman Neon Wolf t-shirt\n",
    "64. psychedelic colorful Howling Galaxy Wolf T-shirt/Colorful Rainbow Animal Print Head Shirt for woman Neon Wolf t-shirt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27949\n",
      "7\n"
     ]
    }
   ],
   "source": [
    "print(data_sorted.shape[0])\n",
    "print(data_sorted.shape[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "indices=[]\n",
    "for i,row in data_sorted.iterrows():\n",
    "    indices.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "stage1_dedupe_asins=[]\n",
    "i=0\n",
    "j=0\n",
    "num_data_points=data_sorted.shape[0]\n",
    "while i<num_data_points and j<num_data_points:\n",
    "    \n",
    "    previous_i=i\n",
    "    \n",
    "    #store the list of words of ith title in a\n",
    "    a=data['title'].loc[indices[i]].split()\n",
    "    \n",
    "    #search for similar title sequentially\n",
    "    j=i+1\n",
    "    while j<num_data_points:\n",
    "        #store the words of jth title in b\n",
    "        b=data['title'].loc[indices[j]].split()\n",
    "        \n",
    "        #store the max length of two strings\n",
    "        length=max(len(a),len(b))\n",
    "        \n",
    "        #count stores the no of words that are present in both titles\n",
    "        count=0\n",
    "        \n",
    "        #counting the no of same words in a,b\n",
    "        for k in itertools.zip_longest(a,b):\n",
    "            if (k[0]==k[1]):\n",
    "                count+=1\n",
    "        \n",
    "        if (length-count)>2:\n",
    "            stage1_dedupe_asins.append(data_sorted['asin'].loc[indices[i]])\n",
    "            \n",
    "            if j==num_data_points-1:\n",
    "                stage1_dedupe_asins.append(data_sorted['asin'].loc[indices[j]])\n",
    "            \n",
    "            i=j\n",
    "            break\n",
    "        else:\n",
    "            j+=1\n",
    "    if previous_i==i:\n",
    "        break\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=data.loc[data['asin'].isin(stage1_dedupe_asins)]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### We removed duplicates which differ only at the end."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of data points :  17593\n"
     ]
    }
   ],
   "source": [
    "print('Number of data points : ',data.shape[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_pickle('pickels/17k_apperal_data')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}