Scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "source": [
    "!pip install selenium\r\n",
    "!pip install parsel\r\n",
    "!pip install pathlib\r\n",
    "!pip install numpy\r\n",
    "!pip install pandas\r\n",
    "!pip install bs4\r\n",
    "!pip install requests"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Requirement already satisfied: selenium in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\selenium-4.0.0rc1-py3.9.egg (4.0.0rc1)\n",
      "Requirement already satisfied: trio-websocket~=0.9 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\trio_websocket-0.9.2-py3.9.egg (from selenium) (0.9.2)\n",
      "Requirement already satisfied: trio~=0.17 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\trio-0.19.0-py3.9.egg (from selenium) (0.19.0)\n",
      "Requirement already satisfied: urllib3[secure]~=1.26 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\urllib3-1.26.6-py3.9.egg (from selenium) (1.26.6)\n",
      "Requirement already satisfied: async-generator>=1.9 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\async_generator-1.10-py3.9.egg (from trio~=0.17->selenium) (1.10)\n",
      "Requirement already satisfied: attrs>=19.2.0 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\attrs-21.2.0-py3.9.egg (from trio~=0.17->selenium) (21.2.0)\n",
      "Requirement already satisfied: cffi>=1.14 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\cffi-1.14.6-py3.9-win-amd64.egg (from trio~=0.17->selenium) (1.14.6)\n",
      "Requirement already satisfied: idna in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\idna-3.2-py3.9.egg (from trio~=0.17->selenium) (3.2)\n",
      "Requirement already satisfied: outcome in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\outcome-1.1.0-py3.9.egg (from trio~=0.17->selenium) (1.1.0)\n",
      "Requirement already satisfied: sniffio in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\sniffio-1.2.0-py3.9.egg (from trio~=0.17->selenium) (1.2.0)\n",
      "Requirement already satisfied: sortedcontainers in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\sortedcontainers-2.4.0-py3.9.egg (from trio~=0.17->selenium) (2.4.0)\n",
      "Requirement already satisfied: pycparser in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\pycparser-2.20-py3.9.egg (from cffi>=1.14->trio~=0.17->selenium) (2.20)\n",
      "Requirement already satisfied: wsproto>=0.14 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\wsproto-1.0.0-py3.9.egg (from trio-websocket~=0.9->selenium) (1.0.0)\n",
      "Requirement already satisfied: certifi in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\certifi-2021.5.30-py3.9.egg (from urllib3[secure]~=1.26->selenium) (2021.5.30)\n",
      "Requirement already satisfied: cryptography>=1.3.4 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\cryptography-3.4.8-py3.9-win-amd64.egg (from urllib3[secure]~=1.26->selenium) (3.4.8)\n",
      "Requirement already satisfied: pyOpenSSL>=0.14 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\pyopenssl-20.0.1-py3.9.egg (from urllib3[secure]~=1.26->selenium) (20.0.1)\n",
      "Requirement already satisfied: six>=1.5.2 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\six-1.16.0-py3.9.egg (from pyOpenSSL>=0.14->urllib3[secure]~=1.26->selenium) (1.16.0)\n",
      "Requirement already satisfied: h11<1,>=0.9.0 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\h11-0.12.0-py3.9.egg (from wsproto>=0.14->trio-websocket~=0.9->selenium) (0.12.0)\n",
      "Requirement already satisfied: parsel in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (1.6.0)\n",
      "Requirement already satisfied: cssselect>=0.9 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from parsel) (1.1.0)\n",
      "Requirement already satisfied: lxml in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\lxml-4.6.3-py3.9-win-amd64.egg (from parsel) (4.6.3)\n",
      "Requirement already satisfied: six>=1.6.0 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\six-1.16.0-py3.9.egg (from parsel) (1.16.0)\n",
      "Requirement already satisfied: w3lib>=1.19.0 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from parsel) (1.22.0)\n",
      "Requirement already satisfied: pathlib in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (1.0.1)\n",
      "Requirement already satisfied: numpy in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (1.21.2)\n",
      "Requirement already satisfied: pandas in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (1.3.2)\n",
      "Requirement already satisfied: pytz>=2017.3 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from pandas) (2021.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from pandas) (2.8.2)\n",
      "Requirement already satisfied: numpy>=1.17.3 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from pandas) (1.21.2)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\six-1.16.0-py3.9.egg (from python-dateutil>=2.7.3->pandas) (1.16.0)\n",
      "Requirement already satisfied: bs4 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (0.0.1)\n",
      "Requirement already satisfied: beautifulsoup4 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from bs4) (4.10.0)\n",
      "Requirement already satisfied: soupsieve>1.2 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from beautifulsoup4->bs4) (2.2.1)\n",
      "Requirement already satisfied: requests in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\requests-2.26.0-py3.9.egg (2.26.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\certifi-2021.5.30-py3.9.egg (from requests) (2021.5.30)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\charset_normalizer-2.0.4-py3.9.egg (from requests) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\idna-3.2-py3.9.egg (from requests) (3.2)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\genext\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\urllib3-1.26.6-py3.9.egg (from requests) (1.26.6)\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "source": [
    "# Given the profile urls scrape skills and other info\r\n",
    "\r\n",
    "from selenium import webdriver\r\n",
    "from parsel import Selector\r\n",
    "import time\r\n",
    "import pathlib\r\n",
    "\r\n",
    "import requests,json,re,datetime\r\n",
    "from bs4 import BeautifulSoup\r\n",
    "import numpy as np\r\n",
    "import pandas as pd\r\n",
    "\r\n"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "source": [
    "chromedriver = str(pathlib.Path().resolve())+'/chromedriver.exe' #Put your chromedriver.exe into current directory as MAC and windows have different system path 😉\r\n",
    "\r\n",
    "_DRIVER_CHROME = webdriver.Chrome(chromedriver)\r\n",
    "\r\n",
    "\r\n",
    "\r\n",
    "_DRIVER_CHROME.get('https://www.linkedin.com/uas/login')\r\n",
    "\r\n",
    "elementID = _DRIVER_CHROME.find_element_by_id('username')\r\n",
    "elementID.send_keys('') # Enter your email between quotes\r\n",
    "\r\n",
    "elementID = _DRIVER_CHROME.find_element_by_id('password')\r\n",
    "elementID.send_keys('') # Enter your password between quotes\r\n",
    "\r\n",
    "elementID.submit()\r\n",
    "time.sleep(30) \r\n"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "C:\\Users\\GeNext\\AppData\\Local\\Temp/ipykernel_11568/4265716530.py:3: DeprecationWarning: executable_path has been deprecated, please pass in a Service object\n",
      "  _DRIVER_CHROME = webdriver.Chrome(chromedriver)\n",
      "C:\\Users\\GeNext\\AppData\\Local\\Temp/ipykernel_11568/4265716530.py:9: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead\n",
      "  elementID = _DRIVER_CHROME.find_element_by_id('username')\n",
      "C:\\Users\\GeNext\\AppData\\Local\\Temp/ipykernel_11568/4265716530.py:12: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead\n",
      "  elementID = _DRIVER_CHROME.find_element_by_id('password')\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "source": [
    "def get_json1(url):\r\n",
    "\t_DRIVER_CHROME.get(url)\r\n",
    "\thtml = _DRIVER_CHROME.page_source\r\n",
    "\tsoup = BeautifulSoup(html , \"html.parser\")\r\n",
    "\t# with open('atharva_soup_find_all_code_result.txt', 'w', encoding=\"utf-8\") as f:\r\n",
    "\t# \tfor item in soup.findAll('code'):\r\n",
    "\t# \t\tf.write(\"%s\\n\" % item)\r\n",
    "\tfor i in soup.findAll('code'):\r\n",
    "\t\tif(re.match(r'^\\n  {\"request\":\"/voyager/api/identity/dash/profiles.*.FullProfileWithEntities.*',i.text)):\r\n",
    "\t\t\tidd=json.loads(i.text)['body']\r\n",
    "\tdata=soup.find_all(attrs={'id':idd})[0]\r\n",
    "\tfor i in data:\r\n",
    "\t\ts=json.loads(i)\r\n",
    "\treturn s['included']\r\n",
    "\t# print(html)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "source": [
    "def get_mm(dd):\r\n",
    "    ff={}\r\n",
    "    current_month=int(datetime.datetime.today().month)  \r\n",
    "    sm=dd['Start Month']\r\n",
    "    em=dd['End Month']\r\n",
    "    #print(sm,em)\r\n",
    "\r\n",
    "    if sm!='No DateRange Start Month Key' or em!='No DateRange End Month Key':\r\n",
    "        if sm!='No Start Month':\r\n",
    "            if em!='No End Month' and em!='No end month':\r\n",
    "                if em!='Present':\r\n",
    "                    if(em<sm):\r\n",
    "                        ff['m']=em-sm+1\r\n",
    "                    else:\r\n",
    "                        ff['m']= np.abs(em-sm)+1\r\n",
    "                else:\r\n",
    "                    ff['m']=current_month-sm+1\r\n",
    "            else:\r\n",
    "                ff['m']=0\r\n",
    "        else:\r\n",
    "                ff['m']= 0\r\n",
    "    else:\r\n",
    "        ff['m']= 0\r\n",
    "    return ff\r\n"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "source": [
    "def get_educ(s1):\r\n",
    "    ls=[]\r\n",
    "    for i in range(len(s1)):\r\n",
    "        d={}\r\n",
    "        if(re.match('com.linkedin.voyager.dash.identity.profile.Education',s1[i]['$type'])):\r\n",
    "            kk=s1[i]\r\n",
    "            #print(kk.keys())\r\n",
    "            if 'degreeName' in kk.keys() and 'schoolName' in kk.keys() and 'fieldOfStudy' in kk.keys() and 'dateRange' in kk.keys():\r\n",
    "                if(kk['degreeName'] is not None):\r\n",
    "                    d['DegreeName']=kk['degreeName']\r\n",
    "                else:\r\n",
    "                    d['DegreeName']='Not Mentioned'\r\n",
    "\r\n",
    "                if(kk['schoolName'] is not None):\r\n",
    "                    d['SchoolName']=kk['schoolName']\r\n",
    "                else:\r\n",
    "                    d['SchoolName']='Not Mentioned School'\r\n",
    "\r\n",
    "                if(kk['fieldOfStudy'] is not None):\r\n",
    "                    d['FieldOfStudy']=kk['fieldOfStudy']\r\n",
    "                else:\r\n",
    "                    d['FieldOfStudy']='Not Mentioned FS'\r\n",
    "                    \r\n",
    "                if kk['dateRange'] is not None:\r\n",
    "                    if 'start' in kk['dateRange'].keys() and kk['dateRange']['start'] is not None:\r\n",
    "                        if 'year' in kk['dateRange']['start'].keys() and kk['dateRange']['start']['year'] is not None:\r\n",
    "                            d['Start Date']=kk['dateRange']['start']['year']\r\n",
    "                            #dd.append(kk['dateRange']['start']['year'])\r\n",
    "                        else:\r\n",
    "                            d['Start Date']='No Start Date'\r\n",
    "                    else:\r\n",
    "                        d['Start Date']='No Start Key'\r\n",
    "                        \r\n",
    "                    if 'end' in kk['dateRange'].keys() and kk['dateRange']['end'] is not None:\r\n",
    "                        if 'year' in kk['dateRange']['end'].keys() and kk['dateRange']['end']['year'] is not None:\r\n",
    "                            d['End Date']=kk['dateRange']['end']['year']\r\n",
    "                        else:\r\n",
    "                            d['End Date']='No end year'\r\n",
    "                    else:\r\n",
    "                    #l=kk['dateRange']['end']['year']='No end year'\r\n",
    "                        d['End Date']='No end key'\r\n",
    "\r\n",
    "\r\n",
    "            #k=sn+\"-\"+dn+\"-\"+fs\r\n",
    "            #ls.append(i['schoolName'])\r\n",
    "            #ls.append(i['degreeName'])\r\n",
    "            #ls.append(i['fieldOfStudy'])\r\n",
    "            ls.append(d)\r\n",
    "    return ls"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "source": [
    "def get_skills(s1):\r\n",
    "    skills=[]\r\n",
    "    for i in range(len(s1)):\r\n",
    "        if(re.match('com.linkedin.voyager.dash.identity.profile.Skill',s1[i]['$type'])):\r\n",
    "            skills.append(s1[i]['name'])\r\n",
    "    return skills"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "source": [
    "# \"$type\":\"com.linkedin.voyager.dash.identity.profile.PhotoFilterEditInfo\"\r\n",
    "# def get_name(s1):\r\n",
    "#     name =\"\"\r\n",
    "#     for i in range(len(s1)):\r\n",
    "#         if(re.match('com.linkedin.voyager.dash.identity.profile.PhotoFilterEditInfo',s1[i]['profilePicture']['photoFilterEditInfo']['$type'])):\r\n",
    "#             name += s1[i]['firstName']\r\n",
    "#             name+=\" \"\r\n",
    "#             name += s1[i]['LastName']\r\n",
    "#     return name\r\n",
    "\r\n",
    "def get_name(s1):\r\n",
    "    name = ''\r\n",
    "    for i in range(len(s1)):\r\n",
    "        if(re.match('com.linkedin.voyager.dash.identity.profile.Profile$',s1[i]['$type'])):\r\n",
    "            if 'summary' in s1[i].keys():\r\n",
    "                name+=s1[i]['firstName']\r\n",
    "                name+=\" \"\r\n",
    "                name+=s1[i]['lastName']\r\n",
    "                return name"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "source": [
    "def get_certificates(s1):\r\n",
    "    certificates=[]\r\n",
    "    for i in range(len(s1)):\r\n",
    "        ddd={}\r\n",
    "        if(re.match('com.linkedin.voyager.dash.identity.profile.Certification',s1[i]['$type'])):\r\n",
    "            kk=s1[i]\r\n",
    "            if 'authority' in kk.keys() and 'name' in kk.keys() and kk['name'] is not None:\r\n",
    "                if kk['authority'] is not None:\r\n",
    "                    ddd['Name']=kk['name']\r\n",
    "                    ddd['Authority']=kk['authority']\r\n",
    "                    #kk=s1[i]['name']+\"-\"+s1[i]['authority']\r\n",
    "                    certificates.append(ddd)\r\n",
    "                else:\r\n",
    "                    ddd['Name']=kk['name']\r\n",
    "                    ddd['Authority']='No Authority'\r\n",
    "                    #kk=s1[i]['name']+\"-\"+s1[i]['authority']\r\n",
    "                    certificates.append(ddd)\r\n",
    "                    #kk=s1[i]['name']+\"-\"+'No Auth'\r\n",
    "                    #certificates.append(kk)\r\n",
    "    return certificates"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "source": [
    "def get_volexp(s1):\r\n",
    "    volexp=[]\r\n",
    "    for i in range(len(s1)):\r\n",
    "        if re.match('com.linkedin.voyager.dash.identity.profile.VolunteerExperience',s1[i]['$type']):\r\n",
    "            d={}\r\n",
    "            kk=s1[i]\r\n",
    "\r\n",
    "            if 'role' in kk.keys() and 'companyName' in kk.keys():\r\n",
    "                if kk['role'] is not None:\r\n",
    "                    d['Role']=kk['role']\r\n",
    "                else:\r\n",
    "                    d['Role']='No Role'\r\n",
    "\r\n",
    "                if kk['companyName'] is not None:\r\n",
    "                    d['Company Name']=kk['companyName']\r\n",
    "\r\n",
    "                else:\r\n",
    "                    d['Company Name']='No Company Name'\r\n",
    "\r\n",
    "\r\n",
    "            if 'description' in kk and kk['description'] is not None:\r\n",
    "                d['Description']=kk['description']\r\n",
    "            else:\r\n",
    "                d['Description']='No Description'\r\n",
    "\r\n",
    "\r\n",
    "            #print(tt,loc,dd,cc)\r\n",
    "            #print('------------------------------------')\r\n",
    "            #Job_details=tt+\"-\"+cc+\"-\"+loc+\"-\"+dd\r\n",
    "            #print(Job_details)\r\n",
    "            volexp.append(d)\r\n",
    "    return volexp"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "source": [
    "def get_projects(s1):\r\n",
    "    projects=[]\r\n",
    "    for i in range(len(s1)):\r\n",
    "        d={}\r\n",
    "        if(re.match('com.linkedin.voyager.dash.identity.profile.Project',s1[i]['$type'])):\r\n",
    "            if 'title' in s1[i] and 'description' in s1[i] and s1[i]['title'] is not None:\r\n",
    "                if (s1[i]['description'] is not None):\r\n",
    "                    d['Title']=s1[i]['title']\r\n",
    "                    d['Description']=s1[i]['description']\r\n",
    "                    #vv=s1[i]['title']+'-'+s1[i]['description']\r\n",
    "                    projects.append(d)\r\n",
    "                else:\r\n",
    "                    d['Title']=s1[i]['title']\r\n",
    "                    d['Description']='No Description'\r\n",
    "                    #vv=s1[i]['title']+'-'+s1[i]['description']\r\n",
    "                    projects.append(d)\r\n",
    "                    #vv=s1[i]['title']+'-'+'No Descc'\r\n",
    "                    #projects.append(vv)\r\n",
    "    return projects\r\n",
    "\r\n",
    "#projects          #com.linkedin.voyager.dash.identity.profile.Honor"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "source": [
    "def get_patents(s1):\r\n",
    "    patents=[]\r\n",
    "    for i in range(len(s1)):\r\n",
    "        d={}\r\n",
    "        if(re.match('com.linkedin.voyager.dash.identity.profile.Patent',s1[i]['$type'])):\r\n",
    "            if 'title' in s1[i].keys() and 'description' in s1[i].keys() and s1[i]['title'] is not None:\r\n",
    "                if (s1[i]['description'] is not None):\r\n",
    "                    d['Title']=s1[i]['title']\r\n",
    "                    d['Description']=s1[i]['title']\r\n",
    "                    #vv=s1[i]['title']+'-'+s1[i]['description']\r\n",
    "                    patents.append(d)\r\n",
    "                else:\r\n",
    "                    d['Title']=s1[i]['title']\r\n",
    "                    d['Description']='No Description'\r\n",
    "                    #vv=s1[i]['title']+'-'+s1[i]['description']\r\n",
    "                    patents.append(d)\r\n",
    "                    #vv=s1[i]['title']+'-'+'No Descc'\r\n",
    "                    #patents.append(vv)\r\n",
    "    return patents\r\n",
    "            #com.linkedin.voyager.dash.identity.profile.Honor"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "source": [
    "def get_honors(s1):\r\n",
    "    honors=[]\r\n",
    "    for i in range(len(s1)):\r\n",
    "        d={}\r\n",
    "        if(re.match('com.linkedin.voyager.dash.identity.profile.Honor',s1[i]['$type'])):\r\n",
    "            kk=s1[i]\r\n",
    "            if 'title' in kk.keys() and 'description' in kk.keys() and kk['title'] is not None:\r\n",
    "                if (kk['description'] is not None):\r\n",
    "                    d['Title']=kk['title']\r\n",
    "                    d['Description']=kk['description']\r\n",
    "                    #vv=s1[i]['title']+'-'+s1[i]['description']\r\n",
    "                    honors.append(d)\r\n",
    "                else:\r\n",
    "                    d['Title']=kk['title']\r\n",
    "                    d['Description']='No Description'\r\n",
    "                    #vv=s1[i]['title']+'-'+s1[i]['description']\r\n",
    "                    honors.append(d)\r\n",
    "                    #vv=s1[i]['title']+'-'+'No Descc'\r\n",
    "                    #honors.append(vv)\r\n",
    "    return honors\r\n",
    " #com.linkedin.voyager.dash.identity.profile.Honor"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "source": [
    "def get_publication(s1):\r\n",
    "    publications=[]\r\n",
    "    for i in range(len(s1)):\r\n",
    "        d={}\r\n",
    "        if(re.match('com.linkedin.voyager.dash.identity.profile.Publication',s1[i]['$type'])):\r\n",
    "            if 'name' in s1[i].keys() and 'publisher' in s1[i].keys() and s1[i]['name'] is not None:\r\n",
    "                if (s1[i]['publisher'] is not None):\r\n",
    "                    d['Name']=s1[i]['name']\r\n",
    "                    d['Publisher']=s1[i]['publisher']\r\n",
    "                    #vv=s1[i]['name']+'-'+s1[i]['publisher']\r\n",
    "                    publications.append(d)\r\n",
    "                else:\r\n",
    "                    d['Name']=s1[i]['name']\r\n",
    "                    d['Publisher']='No Publisher Name'\r\n",
    "                    #vv=s1[i]['name']+'-'+s1[i]['publisher']\r\n",
    "                    publications.append(d)\r\n",
    "                    #vv=s1[i]['name']+'-'+'No Descc'\r\n",
    "                    #publications.append(vv)\r\n",
    "    return publications"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "source": [
    "def get_exp(s1):\r\n",
    "    exps=[]\r\n",
    "    current_year=int(datetime.datetime.today().year)\r\n",
    "    current_month=int(datetime.datetime.today().month)\r\n",
    "    for i in range(len(s1)):\r\n",
    "        d={}\r\n",
    "        if re.match('com.linkedin.voyager.dash.identity.profile.Position$',s1[i]['$type']):\r\n",
    "            kk=s1[i]\r\n",
    "            if 'title' in kk.keys() and 'companyName' in kk.keys() and kk['title'] is not None and 'dateRange' in kk.keys():\r\n",
    "                if kk['companyName'] is not None:\r\n",
    "                    d['Title']=kk['title']\r\n",
    "                    d['Company Name']=kk['companyName']\r\n",
    "                else:\r\n",
    "                    d['Title']=kk['title']\r\n",
    "                    d['Company Name']='No Company Name'\r\n",
    "                \r\n",
    "                if kk['dateRange'] is not None:\r\n",
    "                    if 'start' in kk['dateRange'].keys() and kk['dateRange']['start'] is not None:\r\n",
    "                        if 'year' in kk['dateRange']['start'].keys() and kk['dateRange']['start']['year'] is not None:\r\n",
    "                            d['Start Year']=kk['dateRange']['start']['year']\r\n",
    "                            #dd.append(kk['dateRange']['start']['year'])\r\n",
    "                        else:\r\n",
    "                            d['Start Year']='No Start Year'\r\n",
    "                            \r\n",
    "                        if 'month' in kk['dateRange']['start'].keys() and kk['dateRange']['start']['month'] is not None:\r\n",
    "                            d['Start Month']=kk['dateRange']['start']['month']\r\n",
    "                            #dd.append(kk['dateRange']['start']['year'])\r\n",
    "                        else:\r\n",
    "                            d['Start Month']='No Start Month'\r\n",
    "                             \r\n",
    "                    else:\r\n",
    "                        d['Start Year']='No Start Year Key'\r\n",
    "                        d['Start Month']='No Start Month Key'\r\n",
    "                        \r\n",
    "                    if 'end' in kk['dateRange'].keys() and kk['dateRange']['end'] is not None:\r\n",
    "                        if 'year' in kk['dateRange']['end'].keys() and kk['dateRange']['end']['year'] is not None:\r\n",
    "                            d['End Year']=kk['dateRange']['end']['year']\r\n",
    "                        else:\r\n",
    "                            d['End Year']='No end year'\r\n",
    "                            \r\n",
    "                        \r\n",
    "                        if 'month' in kk['dateRange']['end'].keys() and kk['dateRange']['end']['month'] is not None:\r\n",
    "                            d['End Month']=kk['dateRange']['end']['month']\r\n",
    "                        else:\r\n",
    "                            d['End Month']='No end month'\r\n",
    "                        \r\n",
    "                    else:\r\n",
    "                    #l=kk['dateRange']['end']['year']='No end year'\r\n",
    "                        d['End Year']='Present'\r\n",
    "                        d['End Month']='Present'\r\n",
    "                else:\r\n",
    "                    d['Start Year']='No DateRange Value'\r\n",
    "                    d['End Year']='No DateRange Value'\r\n",
    "                \r\n",
    "            else:\r\n",
    "                d['Title']='No title key'\r\n",
    "                d['Company Name']='No companyName key'\r\n",
    "                d['Start Year']='No DateRange Start Key'\r\n",
    "                d['End Year']='No DateRange End Key'\r\n",
    "                d['Start Month']='No DateRange Start Month Key'\r\n",
    "                d['End Month']='No DateRange End Month Key'\r\n",
    "            #print(d)\r\n",
    "            if d['Title']=='No title key':\r\n",
    "                   continue\r\n",
    "            \r\n",
    "            #if d['Title']!='No Title Key':            \r\n",
    "            if d['Start Year'] is not None and d['Start Year']!='No Start Year' and d['Start Year']!='No DateRange Value' and d['Start Year']!='No DateRange Start Key':\r\n",
    "                if d['End Year']!='No end year':\r\n",
    "                    d['Exp (in months)']=12*int(current_year-d['Start Year'])\r\n",
    "                if d['End Year']!='Present':\r\n",
    "                    d['Exp (in months)']=12*int(d['End Year']-d['Start Year'])\r\n",
    "                else:\r\n",
    "                    d['Exp (in months)']=12*int(current_year-d['Start Year'])\r\n",
    "                #print(d)\r\n",
    "                jjj=get_mm(d)\r\n",
    "                d['Total Exp']=jjj['m']+d['Exp (in months)']\r\n",
    "            \r\n",
    "            if 'geoLocationName' in kk.keys() and kk['geoLocationName'] is not None:\r\n",
    "                d['Location']=kk['geoLocationName']\r\n",
    "            else:\r\n",
    "                d['Location']='No Location'\r\n",
    "\r\n",
    "            if 'description' in kk.keys() and kk['description'] is not None:\r\n",
    "                d['Description']=kk['description']\r\n",
    "            else:\r\n",
    "                d['Description']='No Description'\r\n",
    "            \r\n",
    "            \r\n",
    "            #print(tt,loc,dd,cc)\r\n",
    "            #print('------------------------------------')\r\n",
    "            #Job_details=tt+\"-\"+cc+\"-\"+loc+\"-\"+dd\r\n",
    "            #print(Job_details)\r\n",
    "            #if d['Title']\r\n",
    "            exps.append(d)\r\n",
    "    return exps"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "source": [
    "def get_about(s1):\r\n",
    "    summary=''\r\n",
    "    for i in range(len(s1)):\r\n",
    "        if(re.match('com.linkedin.voyager.dash.identity.profile.Profile$',s1[i]['$type'])):\r\n",
    "            if 'summary' in s1[i].keys() and s1[i]['summary'] is not None:\r\n",
    "                summary=s1[i]['summary']\r\n",
    "                return summary\r\n",
    "    if summary=='':\r\n",
    "        return \"None\""
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "source": [
    "def brain(urls):\r\n",
    "    main_df=pd.DataFrame()\r\n",
    "    for url in urls:\r\n",
    "        print(url)\r\n",
    "        df=pd.DataFrame()\r\n",
    "        s=get_json1(url)\r\n",
    "        # with open('atharva_get_json_output.json', 'w') as fp:\r\n",
    "        #     json.dump(s, fp)\r\n",
    "        df['Name'] = [get_name(s)]\r\n",
    "        df['Profile Url']=[url]\r\n",
    "        df['About']=[get_about(s)]\r\n",
    "        df['Education']=[get_educ(s)]\r\n",
    "        df['Experience']=[get_exp(s)]\r\n",
    "        df['Skills']=[get_skills(s)]\r\n",
    "        df['Certificates']=[get_certificates(s)]\r\n",
    "        df['Vol Exp']=[get_volexp(s)]\r\n",
    "        df['Projects']=[get_projects(s)]\r\n",
    "        df['Patents']=[get_patents(s)]\r\n",
    "        df['Honors']=[get_honors(s)]\r\n",
    "        df['Publications']=[get_publication(s)]\r\n",
    "        main_df=pd.concat([main_df,df],axis=0,ignore_index=True)\r\n",
    "    return main_df\r\n",
    "# \t\t\r\n",
    "\t\t# print(s)\r\n",
    "\t\t#op=open_to_work(s)\r\n",
    "    # if op is not None and op!='dd':\r\n",
    "\t\t\r\n",
    "\r\n",
    "\r\n",
    "    #     \r\n",
    "   \r\n",
    "    #     \r\n",
    "    #         #df=df.from_dict(d)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "import csv\r\n",
    "\r\n",
    "with open('profile_urls.csv', newline='') as fpro:\r\n",
    "    reader = csv.reader(fpro)\r\n",
    "    url_list = [x[0] for x in reader]\r\n",
    "\r\n",
    "print(url_list)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "source": [
    "# urls = url_list\r\n",
    "urls = ['https://www.linkedin.com/in/atharva-parikh-ap07/','https://www.linkedin.com/in/sidhant-khamankar/']\r\n",
    "# linkedin_scrape(urls)\r\n",
    "# get_json1('https://www.linkedin.com/in/atharva-parikh-ap07/')\r\n",
    "pkk=brain(urls)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "https://www.linkedin.com/in/mohit-charkha-98386b159\n",
      "https://www.linkedin.com/in/geervani-reddy\n",
      "https://www.linkedin.com/in/mayur-gotmare-a85870159\n",
      "https://www.linkedin.com/in/rajeshalane\n",
      "https://www.linkedin.com/in/purohitkrishnar\n",
      "https://www.linkedin.com/in/ayushsbansal\n",
      "https://www.linkedin.com/in/riya-loya-a9267715a\n",
      "https://www.linkedin.com/in/pranavpmehta\n",
      "https://www.linkedin.com/in/arnav-vaidya-733a59180\n",
      "https://www.linkedin.com/in/onkar-chavan-588671183\n",
      "https://www.linkedin.com/in/tanmaypro\n",
      "https://www.linkedin.com/in/yashavant-garje-766630177\n",
      "https://www.linkedin.com/in/aboli-rode-195000133\n",
      "https://www.linkedin.com/in/manshreepadale\n",
      "https://www.linkedin.com/in/parag2812\n",
      "https://www.linkedin.com/in/riya-shah-399059218\n",
      "https://www.linkedin.com/in/tasneem-chopdawala-9b9170159\n",
      "https://www.linkedin.com/in/kunal-ambekar-a60553191\n",
      "https://www.linkedin.com/in/dhaval-tanna-604762159\n",
      "https://www.linkedin.com/in/honeytalreja\n",
      "https://www.linkedin.com/in/akash-varma\n",
      "https://www.linkedin.com/in/rathodpratik557\n",
      "https://www.linkedin.com/in/prathamesh-kulkarni-940710155\n",
      "https://www.linkedin.com/in/pranav-wagh-9070b7170\n",
      "https://www.linkedin.com/in/aniruddha-tonge-b7a24917\n",
      "https://www.linkedin.com/in/dikshakatake\n",
      "https://www.linkedin.com/in/aniruddhasonawane8699\n",
      "https://www.linkedin.com/in/neelkamalsb\n",
      "https://www.linkedin.com/in/aishwarya-jamdade-b95a90171\n",
      "https://www.linkedin.com/in/ria-m-02996b155\n",
      "https://www.linkedin.com/in/rohan-shrishrimal-9b4a14156\n",
      "https://www.linkedin.com/in/theshraddhamore\n",
      "https://www.linkedin.com/in/aditya-jagdale-920a0417b\n",
      "https://www.linkedin.com/in/pratik-jagtap-67b869159\n",
      "https://www.linkedin.com/in/ankit-bawanthade-116773159\n",
      "https://www.linkedin.com/in/animesh-more-a63422192\n",
      "https://www.linkedin.com/in/chinmay-mundhada\n",
      "https://www.linkedin.com/in/arti-karad-5a0426149\n",
      "https://www.linkedin.com/in/vishwesh-chaudhari\n",
      "https://www.linkedin.com/in/rakesh-mallesh-82385494\n",
      "https://www.linkedin.com/in/mrunal-vyas-a6a232142\n",
      "https://www.linkedin.com/in/rupesh-kapse-25133918b\n",
      "https://www.linkedin.com/in/rajvi-kothari-891888159\n",
      "https://www.linkedin.com/in/aiman-shivani-b04278190\n",
      "https://www.linkedin.com/in/harikrishna-gonapuri\n",
      "https://www.linkedin.com/in/rutuja-wayase-a52868159\n",
      "https://www.linkedin.com/in/abhishek-khedekar-5a4316192\n",
      "https://www.linkedin.com/in/prajwal-patil-779822177\n",
      "https://www.linkedin.com/in/chetan-patil\n",
      "https://www.linkedin.com/in/aanshi-verma-24b871159\n",
      "https://www.linkedin.com/in/mahesh-dhote-2303\n",
      "https://www.linkedin.com/in/mayur-bhor-32b746170\n",
      "https://www.linkedin.com/in/aishwarya-ulhe\n",
      "https://www.linkedin.com/in/siddhesh-yeole-84b869159\n",
      "https://www.linkedin.com/in/siddhi-retharekar-04173615b\n",
      "https://www.linkedin.com/in/aarya-dhore-170766159\n",
      "https://www.linkedin.com/in/neeraj-pise\n",
      "https://www.linkedin.com/in/saumitra-kulkarni-13a91116a\n",
      "https://www.linkedin.com/in/pallavi-shelke-7636a11b1\n",
      "https://www.linkedin.com/in/nirmiti-gawande-b2a2b6171\n",
      "https://www.linkedin.com/in/jugal-patil-3348ab157\n",
      "https://www.linkedin.com/in/samrudhi-kedari-0a85a2171\n",
      "https://www.linkedin.com/in/vijaykumar-nagre-78b855114\n",
      "https://www.linkedin.com/in/pranavshirude\n",
      "https://www.linkedin.com/in/aishwarya-yeole-0a5765159\n",
      "https://www.linkedin.com/in/sarthak-ladhwe\n",
      "https://www.linkedin.com/in/pranav-bagade-5862761b4\n",
      "https://www.linkedin.com/in/shreepal-gurav-02a6a3b2\n",
      "https://www.linkedin.com/in/mohammed-abdul-umair-966763159\n",
      "https://www.linkedin.com/in/rohit-kumar-shaw-037765159\n",
      "https://www.linkedin.com/in/tanmay-raut\n",
      "https://www.linkedin.com/in/adarsh-khandelwal-bb7477150\n",
      "https://www.linkedin.com/in/shrinidhi-gawande-7378a4176\n",
      "https://www.linkedin.com/in/siddesh-vyavahare-96229b150\n",
      "https://www.linkedin.com/in/atharva-parikh-ap07\n",
      "https://www.linkedin.com/in/namrata-chavan-768974157\n",
      "https://www.linkedin.com/in/shubham-jha-52041915a\n",
      "https://www.linkedin.com/in/devendra-naik-b3b07b125\n",
      "https://www.linkedin.com/in/siddharth-abrol-081685170\n",
      "https://www.linkedin.com/in/viraj-chatane\n",
      "https://www.linkedin.com/in/sopan-kadam-5b994459\n",
      "https://www.linkedin.com/in/sankalp-doshi-037796165\n",
      "https://www.linkedin.com/in/mansi-mane-b8386a159\n",
      "https://www.linkedin.com/in/om-jain-204678191\n",
      "https://www.linkedin.com/in/riya-kotnis-84032815a\n",
      "https://www.linkedin.com/in/rahul-jyoti-508151169\n",
      "https://www.linkedin.com/in/uravi-kariya-245869159\n",
      "https://www.linkedin.com/in/prateek-misal-30271b192\n",
      "https://www.linkedin.com/in/roshan-chavan-a84455200\n",
      "https://www.linkedin.com/in/priyankabirajdar\n",
      "https://www.linkedin.com/in/shreeraj-shivale-90204b196\n",
      "https://www.linkedin.com/in/sakshikangane\n",
      "https://www.linkedin.com/in/ishan-deshpande\n",
      "https://www.linkedin.com/in/sahil-bhaware-b2b771159\n",
      "https://www.linkedin.com/in/prajwal-ujawane-b77951145\n",
      "https://www.linkedin.com/in/abhijeetkmore\n",
      "https://www.linkedin.com/in/akshay-taphare-8885141a1\n",
      "https://www.linkedin.com/in/akanksha-hajare-60b762159\n",
      "https://www.linkedin.com/in/himanshu-panwar-673772159\n",
      "https://www.linkedin.com/in/rajashri-j-829331b3\n",
      "https://www.linkedin.com/in/paritosh-marathe-aa0b21150\n",
      "https://www.linkedin.com/in/harsh-waychal-95b886191\n",
      "https://www.linkedin.com/in/nikita-patil-1b067315a\n",
      "https://www.linkedin.com/in/sahil-parab49\n",
      "https://www.linkedin.com/in/rajvaibhav-rahane\n",
      "https://www.linkedin.com/in/abhishek-supsande-95870616a\n",
      "https://www.linkedin.com/in/abhijeet-sarate-185589154\n",
      "https://www.linkedin.com/in/shriya-bhapkar-50797a180\n",
      "https://www.linkedin.com/in/tejal-lolap\n",
      "https://www.linkedin.com/in/prajwal-dabhade-700740192\n",
      "https://www.linkedin.com/in/pranav-kumbhar-082b5b64\n",
      "https://www.linkedin.com/in/diwanevasant\n",
      "https://www.linkedin.com/in/rutvik-darwatkar-6b8a40193\n",
      "https://www.linkedin.com/in/rittika-dolui-2b4b95132\n",
      "https://www.linkedin.com/in/sohamtupe\n",
      "https://www.linkedin.com/in/harishalhate28\n",
      "https://www.linkedin.com/in/shubhamjaiswal1403\n",
      "https://www.linkedin.com/in/sandeshl98\n",
      "https://www.linkedin.com/in/sumeet-rathod-b85b39159\n",
      "https://www.linkedin.com/in/ali-asgar-fakhruddin-2babba152\n",
      "https://www.linkedin.com/in/radhika-bhoj\n",
      "https://www.linkedin.com/in/anurag-agrawal-674406164\n",
      "https://www.linkedin.com/in/adwait-gunjal-1899a0b1\n",
      "https://www.linkedin.com/in/ajinkya-badave-b145b6134\n",
      "https://www.linkedin.com/in/samarth-datir-406793195\n",
      "https://www.linkedin.com/in/swapnil-ahire\n",
      "https://www.linkedin.com/in/siddhi-maheshwari-ba845a188\n",
      "https://www.linkedin.com/in/shashnagaral\n",
      "https://www.linkedin.com/in/navghareaniket\n",
      "https://www.linkedin.com/in/harsh-shinde-951896135\n",
      "https://www.linkedin.com/in/aditya-khedkar-5847a5193\n",
      "https://www.linkedin.com/in/aditi-avhad-616400184\n",
      "https://www.linkedin.com/in/manjiri-nerurkar-a60b43124\n",
      "https://www.linkedin.com/in/vishal334\n",
      "https://www.linkedin.com/in/nitish-mhaske-497552154\n",
      "https://www.linkedin.com/in/acnahar\n",
      "https://www.linkedin.com/in/thepiyushgiri\n",
      "https://www.linkedin.com/in/nikhil-patil-101766159\n",
      "https://www.linkedin.com/in/omkarkadam158\n",
      "https://www.linkedin.com/in/kajol-kuwad-5122a5155\n",
      "https://www.linkedin.com/in/rishi-shrimali-1aa646191\n",
      "https://www.linkedin.com/in/jayzilpilwar\n",
      "https://www.linkedin.com/in/sheetal-bhosale-3a9770159\n",
      "https://www.linkedin.com/in/yash-mamidwar-b29553195\n",
      "https://www.linkedin.com/in/utkarsh-landge-3759ab192\n",
      "https://www.linkedin.com/in/raunak-jain-035b5010b\n",
      "https://www.linkedin.com/in/pratyaksh-agarwal-393429171\n",
      "https://www.linkedin.com/in/amang2110\n",
      "https://www.linkedin.com/in/mandar-dange-learner\n",
      "https://www.linkedin.com/in/shaila-kale777\n",
      "https://www.linkedin.com/in/pranit-tambe\n",
      "https://www.linkedin.com/in/onkar-kumbhar-008312150\n",
      "https://www.linkedin.com/in/umer-farooque-anjum\n",
      "https://www.linkedin.com/in/bitakeyogesh\n",
      "https://www.linkedin.com/in/mayur-sangale-76087a159\n",
      "https://www.linkedin.com/in/sidhant-khamankar\n",
      "https://www.linkedin.com/in/praveen-hedaoo\n",
      "https://www.linkedin.com/in/ajay-thombare\n",
      "https://www.linkedin.com/in/ankitbhadoriya\n",
      "https://www.linkedin.com/in/aditya-bakare\n",
      "https://www.linkedin.com/in/akash-nainani\n",
      "https://www.linkedin.com/in/panjab-t-kale-a64937169\n",
      "https://www.linkedin.com/in/saurav-hole\n",
      "https://www.linkedin.com/in/manasdodya\n",
      "https://www.linkedin.com/in/jai-lulla\n",
      "https://www.linkedin.com/in/soniya-chavan-22346b163\n",
      "https://www.linkedin.com/in/nirbhay-patil\n",
      "https://www.linkedin.com/in/shriyash-khorate-9b071b152\n",
      "https://www.linkedin.com/in/avez-shaikh\n",
      "https://www.linkedin.com/in/dhananjay-rashinkar-a12678193\n",
      "https://www.linkedin.com/in/rohan-rathi-a22224171\n",
      "https://www.linkedin.com/in/dhiraj-pote-6b7902191\n",
      "https://www.linkedin.com/in/prachi-bhavsar-ba781b179\n",
      "https://www.linkedin.com/in/rutuja-chaudhary-113773159\n",
      "https://www.linkedin.com/in/avinashmawle\n",
      "https://www.linkedin.com/in/pratik-jagade-485347194\n",
      "https://www.linkedin.com/in/mamata-singh-940a7b175\n",
      "https://www.linkedin.com/in/shriram-salunke-992303198\n",
      "https://www.linkedin.com/in/jeet-desai-90a30319b\n",
      "https://www.linkedin.com/in/dhruv-kolhatkar-0b12b8198\n",
      "https://www.linkedin.com/in/yogesh-diwate-6365b5197\n",
      "https://www.linkedin.com/in/piyusha-deshmukh-4b2889157\n",
      "https://www.linkedin.com/in/yash-kolhe-b715b815b\n",
      "https://www.linkedin.com/in/mrunal-vaidhya-42b32415a\n",
      "https://www.linkedin.com/in/sumedh-kulkarni-840ab118b\n",
      "https://www.linkedin.com/in/vedant-bhatambrekar-35a870159\n"
     ]
    },
    {
     "output_type": "error",
     "ename": "UnboundLocalError",
     "evalue": "local variable 'idd' referenced before assignment",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mUnboundLocalError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_11568/2747368752.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;31m# linkedin_scrape(urls)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;31m# get_json1('https://www.linkedin.com/in/atharva-parikh-ap07/')\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mpkk\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murls\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_11568/4292174043.py\u001b[0m in \u001b[0;36mbrain\u001b[1;34m(urls)\u001b[0m\n\u001b[0;32m      4\u001b[0m         \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m         \u001b[0mdf\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 6\u001b[1;33m         \u001b[0ms\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mget_json1\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      7\u001b[0m         \u001b[1;31m# with open('atharva_get_json_output.json', 'w') as fp:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m         \u001b[1;31m#     json.dump(s, fp)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_11568/1977796909.py\u001b[0m in \u001b[0;36mget_json1\u001b[1;34m(url)\u001b[0m\n\u001b[0;32m      9\u001b[0m                 \u001b[1;32mif\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mre\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmatch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mr'^\\n  {\"request\":\"/voyager/api/identity/dash/profiles.*.FullProfileWithEntities.*'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m                         \u001b[0midd\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mjson\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'body'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m         \u001b[0mdata\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msoup\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_all\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mattrs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m{\u001b[0m\u001b[1;34m'id'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0midd\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     12\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m                 \u001b[0ms\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mjson\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mUnboundLocalError\u001b[0m: local variable 'idd' referenced before assignment"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "source": [
    "pkk"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Profile Url</th>\n",
       "      <th>About</th>\n",
       "      <th>Education</th>\n",
       "      <th>Experience</th>\n",
       "      <th>Skills</th>\n",
       "      <th>Certificates</th>\n",
       "      <th>Vol Exp</th>\n",
       "      <th>Projects</th>\n",
       "      <th>Patents</th>\n",
       "      <th>Honors</th>\n",
       "      <th>Publications</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Atharva Parikh</td>\n",
       "      <td>https://www.linkedin.com/in/atharva-parikh-ap07/</td>\n",
       "      <td>I am a TY Information Technology student at VI...</td>\n",
       "      <td>[{'DegreeName': 'TY btech', 'SchoolName': 'Vis...</td>\n",
       "      <td>[{'Title': 'Jt. Head Treasurer', 'Company Name...</td>\n",
       "      <td>[MySQL, Python (Programming Language), HTML, M...</td>\n",
       "      <td>[{'Name': 'Reading Financial Statements', 'Aut...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[{'Title': 'Low code No code applications', 'D...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Sidhant Khamankar</td>\n",
       "      <td>https://www.linkedin.com/in/sidhant-khamankar/</td>\n",
       "      <td>Looking for Software Engineering Intern role.\\...</td>\n",
       "      <td>[{'DegreeName': 'Not Mentioned', 'SchoolName':...</td>\n",
       "      <td>[{'Title': 'Expert', 'Company Name': 'Chegg In...</td>\n",
       "      <td>[Communication, Programming, HTML5, Cascading ...</td>\n",
       "      <td>[{'Name': 'AMCAT Certified Software Developmen...</td>\n",
       "      <td>[{'Role': 'Associate Designing Head', 'Company...</td>\n",
       "      <td>[{'Title': 'Tic Tac Toe Using C', 'Description...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[{'Title': '2nd Runner up in Scitech VIIT annu...</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                Name                                       Profile Url  \\\n",
       "0     Atharva Parikh  https://www.linkedin.com/in/atharva-parikh-ap07/   \n",
       "1  Sidhant Khamankar    https://www.linkedin.com/in/sidhant-khamankar/   \n",
       "\n",
       "                                               About  \\\n",
       "0  I am a TY Information Technology student at VI...   \n",
       "1  Looking for Software Engineering Intern role.\\...   \n",
       "\n",
       "                                           Education  \\\n",
       "0  [{'DegreeName': 'TY btech', 'SchoolName': 'Vis...   \n",
       "1  [{'DegreeName': 'Not Mentioned', 'SchoolName':...   \n",
       "\n",
       "                                          Experience  \\\n",
       "0  [{'Title': 'Jt. Head Treasurer', 'Company Name...   \n",
       "1  [{'Title': 'Expert', 'Company Name': 'Chegg In...   \n",
       "\n",
       "                                              Skills  \\\n",
       "0  [MySQL, Python (Programming Language), HTML, M...   \n",
       "1  [Communication, Programming, HTML5, Cascading ...   \n",
       "\n",
       "                                        Certificates  \\\n",
       "0  [{'Name': 'Reading Financial Statements', 'Aut...   \n",
       "1  [{'Name': 'AMCAT Certified Software Developmen...   \n",
       "\n",
       "                                             Vol Exp  \\\n",
       "0                                                 []   \n",
       "1  [{'Role': 'Associate Designing Head', 'Company...   \n",
       "\n",
       "                                            Projects Patents  \\\n",
       "0  [{'Title': 'Low code No code applications', 'D...      []   \n",
       "1  [{'Title': 'Tic Tac Toe Using C', 'Description...      []   \n",
       "\n",
       "                                              Honors Publications  \n",
       "0                                                 []           []  \n",
       "1  [{'Title': '2nd Runner up in Scitech VIIT annu...           []  "
      ]
     },
     "metadata": {},
     "execution_count": 29
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "source": [
    "# pkk['Education'][0]"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[{'DegreeName': 'TY btech',\n",
       "  'SchoolName': 'Vishwakarma Institute of Information Technology',\n",
       "  'FieldOfStudy': 'Information Technology',\n",
       "  'Start Date': 2019,\n",
       "  'End Date': 2023}]"
      ]
     },
     "metadata": {},
     "execution_count": 19
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "source": [
    "# pkk['Experience'][0]"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[{'Title': 'Jt. Head Treasurer',\n",
       "  'Company Name': 'Information Technocrats Students Forum,VIIT',\n",
       "  'Start Year': 2020,\n",
       "  'Start Month': 8,\n",
       "  'End Year': 2021,\n",
       "  'End Month': 8,\n",
       "  'Exp (in months)': 12,\n",
       "  'Total Exp': 13,\n",
       "  'Location': 'No Location',\n",
       "  'Description': 'No Description'},\n",
       " {'Title': 'Sponsorship Team Member',\n",
       "  'Company Name': 'IoT Forum VIIT',\n",
       "  'Start Year': 2020,\n",
       "  'Start Month': 8,\n",
       "  'End Year': 2021,\n",
       "  'End Month': 1,\n",
       "  'Exp (in months)': 12,\n",
       "  'Total Exp': 6,\n",
       "  'Location': 'Pune, Maharashtra, India',\n",
       "  'Description': 'No Description'},\n",
       " {'Title': 'Core Committee Member',\n",
       "  'Company Name': 'Machine Learning Forum VIIT',\n",
       "  'Start Year': 2020,\n",
       "  'Start Month': 8,\n",
       "  'End Year': 2021,\n",
       "  'End Month': 8,\n",
       "  'Exp (in months)': 12,\n",
       "  'Total Exp': 13,\n",
       "  'Location': 'No Location',\n",
       "  'Description': 'No Description'},\n",
       " {'Title': 'Brand Ambassador',\n",
       "  'Company Name': 'GradsKey',\n",
       "  'Start Year': 2020,\n",
       "  'Start Month': 9,\n",
       "  'End Year': 2021,\n",
       "  'End Month': 8,\n",
       "  'Exp (in months)': 12,\n",
       "  'Total Exp': 12,\n",
       "  'Location': 'No Location',\n",
       "  'Description': 'No Description'}]"
      ]
     },
     "metadata": {},
     "execution_count": 20
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "source": [
    "# pkk['Skills'][0]"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['MySQL',\n",
       " 'Python (Programming Language)',\n",
       " 'HTML',\n",
       " 'Machine Learning',\n",
       " 'Object-Oriented Programming (OOP)',\n",
       " 'Data Structures',\n",
       " 'SQL',\n",
       " 'C (Programming Language)',\n",
       " 'Microsoft PowerPoint',\n",
       " 'Natural Language Processing (NLP)',\n",
       " 'JavaScript',\n",
       " 'Flutter',\n",
       " 'Time Management',\n",
       " 'Algorithms',\n",
       " 'C++',\n",
       " 'Microsoft Excel',\n",
       " 'Microsoft Word',\n",
       " 'Interpersonal Skills']"
      ]
     },
     "metadata": {},
     "execution_count": 21
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "source": [
    "# pkk['Certificates'][0]"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[{'Name': 'Reading Financial Statements',\n",
       "  'Authority': 'Corporate Finance Institute® (CFI)'},\n",
       " {'Name': 'Fundamentals of digital marketing',\n",
       "  'Authority': 'Google Digital Unlocked'},\n",
       " {'Name': 'Excel Skills for Business Specialization',\n",
       "  'Authority': 'Macquarie University'},\n",
       " {'Name': 'Data Structures and Algorithms', 'Authority': 'GradsKey'},\n",
       " {'Name': 'FUNDAMENTALS OF DEEP LEARNING', 'Authority': 'NVIDIA'},\n",
       " {'Name': 'FUNDAMENTALS OF ACCELERATED DATA SCIENCE WITH RAPIDS',\n",
       "  'Authority': 'NVIDIA'},\n",
       " {'Name': 'Financial Markets (with Honors)', 'Authority': 'Yale University'},\n",
       " {'Name': 'Data Science Professional Certificate', 'Authority': 'IBM'}]"
      ]
     },
     "metadata": {},
     "execution_count": 22
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "source": [
    "# pkk['Projects'][0]"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[{'Title': 'Low code No code applications',\n",
       "  'Description': 'Built 3 applications on retool. Learnt about creating dashboards, integrating APIs (Twilio and Slack) and handling JSON objects.'},\n",
       " {'Title': 'Healthcare Appointment Scheduling Application',\n",
       "  'Description': 'Handled Project Flow management and Implementation of Trie Data Structure for searching doctors.'},\n",
       " {'Title': 'Car Rental Application in C++',\n",
       "  'Description': 'Learnt about OOP in C++ and file handling.'}]"
      ]
     },
     "metadata": {},
     "execution_count": 23
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "source": [
    "# pkk['Vol Exp'][0]"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "metadata": {},
     "execution_count": 24
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "source": [
    "# pkk['Honors'][0]"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "metadata": {},
     "execution_count": 25
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "source": [
    "# pkk['About'][0]"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'I am a TY Information Technology student at VIIT Pune.\\nI have keen interest in Data Science 📊 & Finance 📈📉.\\nExploring opportunities...'"
      ]
     },
     "metadata": {},
     "execution_count": 26
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "source": [
    "# pkk['Name'][0]"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'Atharva Parikh'"
      ]
     },
     "metadata": {},
     "execution_count": 27
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "source": [
    "pkk.to_csv('all_viit.csv')"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [],
   "outputs": [],
   "metadata": {}
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.9.0 64-bit"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  },
  "interpreter": {
   "hash": "b8bfa9037174bfc6b4b490744694e7f8266e6abcd8297a22515c1aae04530e12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}