crawler.py

import urllib.request
import urllib
from lxml.html import fromstring
import os
import requests


class Crawler:
    def __init__(self, base_url):
        self.base_url = base_url
        self.file_name = None
        self.xpath_selector = None
        self.page = None
        self.sess = requests.Session()

        if not os.path.isdir("downloads"):
            os.mkdir("downloads")

    def download_file(self, url):
        urllib.request.urlretrieve(url, "downloads/" + self.file_name)

    def get_document(self, url):
        self.page = self.sess.get(url)
        self.xpath_selector = fromstring(self.page.content)

    def follow_redirect(self, url):
        self.page = self.sess.get(url, allow_redirects=True)
        self.xpath_selector = fromstring(self.page.content)
        return self.page.url