From 2d49407954dede0777bf857b0bc6d2e4a08481a5 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 9 Jun 2017 20:56:50 +0500 Subject: [PATCH] drop Python 2 support --- README.rst | 2 +- docs/api.rst | 4 +- docs/index.rst | 2 +- requirements.txt | 3 +- setup.py | 2 - splash/argument_cache.py | 17 --- splash/browser_tab.py | 20 ++-- splash/conftest.py | 1 - splash/cookies.py | 1 - splash/exceptions.py | 3 - splash/har/__init__.py | 2 - splash/har/log.py | 10 +- splash/har/qt.py | 18 ++-- splash/har/schema.py | 3 - splash/har/utils.py | 1 - splash/har_builder.py | 1 - splash/html_element.py | 1 - splash/kernel/__init__.py | 1 - splash/kernel/__main__.py | 1 - splash/kernel/completer.py | 1 - splash/kernel/errors.py | 1 - splash/kernel/inspections.py | 1 - splash/kernel/kernel.py | 6 +- splash/kernel/kernelbase.py | 1 - splash/kernel/lua_parser.py | 1 - splash/lua.py | 11 +- splash/lua_runner.py | 4 +- splash/lua_runtime.py | 1 - splash/network_manager.py | 1 - splash/pool.py | 1 - splash/proxy.py | 10 +- splash/qtrender.py | 4 +- splash/qtrender_image.py | 7 +- splash/qtrender_lua.py | 16 ++- splash/qtutils.py | 5 +- splash/qwebpage.py | 6 +- splash/render_options.py | 17 ++- splash/request_middleware.py | 10 +- splash/resources.py | 4 +- splash/response_middleware.py | 1 - splash/server.py | 1 - splash/tests/conftest.py | 3 +- splash/tests/mockserver.py | 3 +- splash/tests/stress.py | 11 +- splash/tests/test_argument_caching.py | 1 - splash/tests/test_callback_proxy.py | 1 - splash/tests/test_client_disconnects.py | 8 +- splash/tests/test_completer.py | 1 - splash/tests/test_execute.py | 103 +++++++------------ splash/tests/test_execute_callbacks.py | 14 +-- splash/tests/test_execute_element.py | 4 +- splash/tests/test_execute_emulation.py | 1 - splash/tests/test_execute_libraries.py | 1 - splash/tests/test_execute_request_filters.py | 2 - splash/tests/test_har.py | 1 - splash/tests/test_jsonpost.py | 40 +++---- splash/tests/test_jupyter.py | 2 - splash/tests/test_lua_parser.py | 1 - splash/tests/test_lua_utils.py | 1 - splash/tests/test_proxy.py | 1 - splash/tests/test_redirects.py | 8 +- splash/tests/test_render.py | 4 +- splash/tests/test_request_filters.py | 1 - splash/tests/test_response_tracking.py | 1 - splash/tests/test_runjs.py | 1 - splash/tests/test_ui.py | 1 - splash/tests/test_utils.py | 1 - splash/utils.py | 11 +- splash/xvfb.py | 1 - 69 files changed, 138 insertions(+), 293 deletions(-) diff --git a/README.rst b/README.rst index 49a543ad7..7c468c581 100644 --- a/README.rst +++ b/README.rst @@ -15,7 +15,7 @@ Splash - A javascript rendering service :target: https://gitter.im/scrapinghub/splash Splash is a javascript rendering service with an HTTP API. It's a lightweight -browser with an HTTP API, implemented in Python using Twisted and QT. +browser with an HTTP API, implemented in Python 3 using Twisted and QT5. It's fast, lightweight and state-less which makes it easy to distribute. diff --git a/docs/api.rst b/docs/api.rst index a27af945f..1eaab5255 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -111,7 +111,7 @@ allowed_content_types : string : optional Comma-separated list of allowed content types. If present, Splash will abort any request if the response's content type doesn't match any of the content types in this list. - Wildcards are supported using the `fnmatch `_ + Wildcards are supported using the `fnmatch `_ syntax. .. _arg-forbidden-content-types: @@ -120,7 +120,7 @@ forbidden_content_types : string : optional Comma-separated list of forbidden content types. If present, Splash will abort any request if the response's content type matches any of the content types in this list. - Wildcards are supported using the `fnmatch `_ + Wildcards are supported using the `fnmatch `_ syntax. .. _arg-viewport: diff --git a/docs/index.rst b/docs/index.rst index 8f025c43c..365138b74 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,7 +3,7 @@ Splash - A javascript rendering service ======================================= Splash is a javascript rendering service. It's a lightweight web browser -with an HTTP API, implemented in Python using Twisted and QT. The (twisted) +with an HTTP API, implemented in Python 3 using Twisted and QT5. The (twisted) QT reactor is used to make the service fully asynchronous allowing to take advantage of webkit concurrency via QT main loop. Some of Splash features: diff --git a/requirements.txt b/requirements.txt index a513091ec..cd2ac4ceb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -# install PyQt5 (Splash is tested on PyQT 5.5.1) +# install PyQt5 (Splash is tested on PyQT 5.8) # and the following packages: twisted >= 15.5.0, < 16.3.0 qt5reactor @@ -7,7 +7,6 @@ adblockparser >= 0.5 https://github.com/sunu/pyre2/archive/c610be52c3b5379b257d56fc0669d022fd70082a.zip#egg=re2 xvfbwrapper Pillow -six # for scripting support lupa >= 1.3 diff --git a/setup.py b/setup.py index be7981787..9972cd6ef 100755 --- a/setup.py +++ b/setup.py @@ -68,8 +68,6 @@ def get_version(): ], 'classifiers': [ 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', diff --git a/splash/argument_cache.py b/splash/argument_cache.py index d3f9e2c52..06a449256 100644 --- a/splash/argument_cache.py +++ b/splash/argument_cache.py @@ -1,26 +1,9 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import json import hashlib from collections import OrderedDict -# Add move_to_end method for python 2.7 -# See https://github.com/twitter/commons/blob/master/src/python/twitter/common/collections/ordereddict.py#L284 -if not hasattr(OrderedDict, 'move_to_end'): - def move_to_end(self, key, last=True): - link_prev, link_next, key = link = self._OrderedDict__map[key] - link_prev[1] = link_next - link_next[0] = link_prev - root = self._OrderedDict__root - if last: - last = root[0] - link[0] = last - link[1] = root - last[1] = root[0] = link - OrderedDict.move_to_end = move_to_end - - class ArgumentCache(object): """ >>> cache = ArgumentCache() diff --git a/splash/browser_tab.py b/splash/browser_tab.py index 598dbde97..3a1e82b24 100644 --- a/splash/browser_tab.py +++ b/splash/browser_tab.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import base64 import functools import os @@ -13,7 +12,6 @@ from PyQt5.QtWidgets import QApplication from twisted.internet import defer from twisted.python import log -import six from splash import defaults from splash.har.qt import cookies2har @@ -359,7 +357,7 @@ def add_cookie(self, cookie): @property def url(self): """ Current URL """ - return six.text_type(self.web_page.mainFrame().url().toString()) + return str(self.web_page.mainFrame().url().toString()) def go(self, url, callback, errback, baseurl=None, http_method='GET', body=None, headers=None): @@ -597,7 +595,7 @@ def _cancel_all_timers(self): callback_proxy.use_up() def _on_url_changed(self, url): - self.web_page.har.store_redirect(six.text_type(url.toString())) + self.web_page.har.store_redirect(str(url.toString())) self._cancel_timers(self._timers_to_cancel_on_redirect) def _process_js_result(self, obj, allow_dom): @@ -942,20 +940,20 @@ def last_http_status(self): def _frame_to_dict(self, frame, children=True, html=True): g = frame.geometry() res = { - "url": six.text_type(frame.url().toString()), - "requestedUrl": six.text_type(frame.requestedUrl().toString()), + "url": str(frame.url().toString()), + "requestedUrl": str(frame.requestedUrl().toString()), "geometry": (g.x(), g.y(), g.width(), g.height()), - "title": six.text_type(frame.title()) + "title": str(frame.title()) } if html: - res["html"] = six.text_type(frame.toHtml()) + res["html"] = str(frame.toHtml()) if children: res["childFrames"] = [ self._frame_to_dict(f, True, html) for f in frame.childFrames() ] - res["frameName"] = six.text_type(frame.frameName()) + res["frameName"] = str(frame.frameName()) return res @@ -1189,7 +1187,7 @@ def __init__(self, parent=None): @pyqtSlot(str) def log(self, message): - self.messages.append(six.text_type(message)) + self.messages.append(str(message)) class _BrowserTabLogger(object): @@ -1239,7 +1237,7 @@ def log(self, message, min_level=None): if min_level is not None and self.verbosity < min_level: return - if isinstance(message, six.text_type): + if isinstance(message, str): message = message.encode('unicode-escape').decode('ascii') message = "[%s] %s" % (self.uid, message) diff --git a/splash/conftest.py b/splash/conftest.py index 2a20cc38a..450c17157 100644 --- a/splash/conftest.py +++ b/splash/conftest.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import from splash import lua collect_ignore = [] diff --git a/splash/cookies.py b/splash/cookies.py index 1ae0e66e7..dec63c259 100644 --- a/splash/cookies.py +++ b/splash/cookies.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import from PyQt5.QtCore import QDateTime, Qt from PyQt5.QtNetwork import QNetworkRequest, QNetworkCookie, QNetworkCookieJar diff --git a/splash/exceptions.py b/splash/exceptions.py index ab962a6a3..69ed73a4f 100644 --- a/splash/exceptions.py +++ b/splash/exceptions.py @@ -1,7 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import - - class BadOption(Exception): """ Incorrect HTTP API arguments """ pass diff --git a/splash/har/__init__.py b/splash/har/__init__.py index 80954b904..16ec5a5be 100644 --- a/splash/har/__init__.py +++ b/splash/har/__init__.py @@ -1,4 +1,2 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import - from .utils import get_duration, format_datetime diff --git a/splash/har/log.py b/splash/har/log.py index e09137689..0d5d31986 100644 --- a/splash/har/log.py +++ b/splash/har/log.py @@ -1,12 +1,10 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import from collections import namedtuple from datetime import datetime import splash from PyQt5.QtCore import PYQT_VERSION_STR, QT_VERSION_STR from PyQt5.QtWebKit import qWebKitVersion -import six from .utils import get_duration, format_datetime, cleaned_har_entry @@ -48,11 +46,11 @@ def has_entry(self, req_id): def store_url(self, url): """ Call this method when URL is changed. """ - self.events.append(HarEvent(HAR_URL_CHANGED, six.text_type(url))) + self.events.append(HarEvent(HAR_URL_CHANGED, str(url))) def store_title(self, title): """ Call this method when page title is changed. """ - self.events.append(HarEvent(HAR_TITLE_CHANGED, six.text_type(title))) + self.events.append(HarEvent(HAR_TITLE_CHANGED, str(title))) def store_timing(self, name): """ @@ -86,12 +84,12 @@ def todict(self): def _get_browser(self): return { "name": "QWebKit", - "version": six.text_type(qWebKitVersion()), + "version": str(qWebKitVersion()), "comment": "PyQt %s, Qt %s" % (PYQT_VERSION_STR, QT_VERSION_STR), } def _empty_page(self, page_id, started_dt): - if not isinstance(started_dt, six.string_types): + if not isinstance(started_dt, str): started_dt = format_datetime(started_dt) return { diff --git a/splash/har/qt.py b/splash/har/qt.py index 075c706bb..9dae3cac3 100644 --- a/splash/har/qt.py +++ b/splash/har/qt.py @@ -3,12 +3,10 @@ Module with helper utilities to serialize QWebkit objects to HAR. See http://www.softwareishard.com/blog/har-12-spec/. """ -from __future__ import absolute_import import base64 from PyQt5.QtCore import Qt, QVariant, QUrlQuery from PyQt5.QtNetwork import QNetworkRequest -import six from splash.qtutils import ( REQUEST_ERRORS_SHORT, @@ -62,9 +60,9 @@ def cookie2har(cookie): cookie = { "name": qt_to_bytes(cookie.name()).decode('utf8', 'replace'), "value": qt_to_bytes(cookie.value()).decode('utf8', 'replace'), - "path": six.text_type(cookie.path()), - "domain": six.text_type(cookie.domain()), - "expires": six.text_type(cookie.expirationDate().toString(Qt.ISODate)), + "path": str(cookie.path()), + "domain": str(cookie.domain()), + "expires": str(cookie.expirationDate().toString(Qt.ISODate)), "httpOnly": cookie.isHttpOnly(), "secure": cookie.isSecure(), } @@ -75,7 +73,7 @@ def cookie2har(cookie): def querystring2har(url): return [ - {"name": six.text_type(name), "value": six.text_type(value)} + {"name": str(name), "value": str(value)} for name, value in QUrlQuery(url).queryItems() ] @@ -105,7 +103,7 @@ def reply2har(reply, content=None): content_type = reply.header(QNetworkRequest.ContentTypeHeader) if content_type is not None: - res["content"]["mimeType"] = six.text_type(content_type) + res["content"]["mimeType"] = str(content_type) content_length = reply.header(QNetworkRequest.ContentLengthHeader) if content_length is not None: @@ -120,7 +118,7 @@ def reply2har(reply, content=None): status_text = reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute) if status_text is not None: - if not isinstance(status_text, six.text_type): + if not isinstance(status_text, str): status_text = qt_to_bytes(status_text).decode('latin1') res['statusText'] = status_text else: @@ -128,7 +126,7 @@ def reply2har(reply, content=None): redirect_url = reply.attribute(QNetworkRequest.RedirectionTargetAttribute) if redirect_url is not None: - res["redirectURL"] = six.text_type(redirect_url.toString()) + res["redirectURL"] = str(redirect_url.toString()) else: res["redirectURL"] = "" @@ -144,7 +142,7 @@ def request2har(request, operation, outgoing_data=None): """ Serialize QNetworkRequest to HAR. """ return { "method": OPERATION_NAMES.get(operation, '?'), - "url": six.text_type(request.url().toString()), + "url": str(request.url().toString()), "httpVersion": "HTTP/1.1", "cookies": request_cookies2har(request), "queryString": querystring2har(request.url()), diff --git a/splash/har/schema.py b/splash/har/schema.py index d115cda6e..d901e1990 100644 --- a/splash/har/schema.py +++ b/splash/har/schema.py @@ -2,9 +2,6 @@ """ Module for validating HAR data. Uses official HAR JSON Schema. """ -from __future__ import absolute_import - - def validate(instance): """ Validate HAR data; raise an exception if it is invalid """ validator = get_validator() diff --git a/splash/har/utils.py b/splash/har/utils.py index 7cb004631..cf1d84d9a 100644 --- a/splash/har/utils.py +++ b/splash/har/utils.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import base64 from operator import itemgetter import itertools diff --git a/splash/har_builder.py b/splash/har_builder.py index a87e1f2eb..c9a2e469b 100644 --- a/splash/har_builder.py +++ b/splash/har_builder.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import copy from datetime import datetime diff --git a/splash/html_element.py b/splash/html_element.py index a7ccc8620..67e99138b 100644 --- a/splash/html_element.py +++ b/splash/html_element.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import from functools import wraps from splash.exceptions import DOMError diff --git a/splash/kernel/__init__.py b/splash/kernel/__init__.py index 139759b65..40a96afc6 100644 --- a/splash/kernel/__init__.py +++ b/splash/kernel/__init__.py @@ -1,2 +1 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import diff --git a/splash/kernel/__main__.py b/splash/kernel/__main__.py index 797277c95..e742727d1 100644 --- a/splash/kernel/__main__.py +++ b/splash/kernel/__main__.py @@ -9,7 +9,6 @@ $ python -m splash.kernel """ -from __future__ import absolute_import import sys from .kernel import start, install diff --git a/splash/kernel/completer.py b/splash/kernel/completer.py index 181ef7885..c3e5de9a0 100644 --- a/splash/kernel/completer.py +++ b/splash/kernel/completer.py @@ -2,7 +2,6 @@ """ Autocompleter for Lua code. """ -from __future__ import absolute_import import string from splash.utils import dedupe, to_unicode diff --git a/splash/kernel/errors.py b/splash/kernel/errors.py index c29d35b8d..19d3d56a6 100644 --- a/splash/kernel/errors.py +++ b/splash/kernel/errors.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import lupa from splash.exceptions import ScriptError diff --git a/splash/kernel/inspections.py b/splash/kernel/inspections.py index e03a6f38a..56279aed8 100644 --- a/splash/kernel/inspections.py +++ b/splash/kernel/inspections.py @@ -2,7 +2,6 @@ """ Inspections for Lua code. """ -from __future__ import absolute_import import os import glob import json diff --git a/splash/kernel/kernel.py b/splash/kernel/kernel.py index 02db3c54f..94dc0edb4 100644 --- a/splash/kernel/kernel.py +++ b/splash/kernel/kernel.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import os -import six import sys import lupa @@ -30,7 +28,7 @@ def install(user=True): """ Install IPython kernel specification """ - name = 'splash-py2' if six.PY2 else 'splash-py3' + name = 'splash-py3' folder = os.path.join(os.path.dirname(__file__), 'kernels', name) install_kernel_spec(folder, kernel_name="splash", user=user, replace=True) @@ -168,7 +166,7 @@ def done(result): reply, result, ct = result if result: data = { - 'text/plain': result if isinstance(result, six.text_type) else str(result), + 'text/plain': result if isinstance(result, str) else str(result), } if isinstance(result, BinaryCapsule): if result.content_type in {'image/png', 'image/jpeg'}: diff --git a/splash/kernel/kernelbase.py b/splash/kernel/kernelbase.py index 3a85e6388..1e5510964 100644 --- a/splash/kernel/kernelbase.py +++ b/splash/kernel/kernelbase.py @@ -3,7 +3,6 @@ Refactored from IPython.kernel.zmq.kernelbase.Kernel with async execution support. See https://github.com/ipython/ipython/pull/7713. """ -from __future__ import absolute_import import functools import time import sys diff --git a/splash/kernel/lua_parser.py b/splash/kernel/lua_parser.py index d448d564f..a74896418 100644 --- a/splash/kernel/lua_parser.py +++ b/splash/kernel/lua_parser.py @@ -2,7 +2,6 @@ """ Parser for a subset of Lua, useful for autocompletion. """ -from __future__ import absolute_import import string from operator import attrgetter from collections import namedtuple diff --git a/splash/lua.py b/splash/lua.py index a2e319cd6..882ec0d15 100644 --- a/splash/lua.py +++ b/splash/lua.py @@ -1,18 +1,17 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, division import re import datetime -from splash.utils import to_bytes, to_unicode from twisted.python import log try: import lupa except ImportError: lupa = None -import six +from splash.utils import to_bytes, to_unicode from splash.exceptions import ScriptError + _supported = None _lua = None @@ -148,7 +147,7 @@ def l2p(obj, depth): if isinstance(obj, dict): return { l2p(key, depth-1): l2p(value, depth-1) - for key, value in six.iteritems(obj) + for key, value in obj.items() } if isinstance(obj, list): @@ -230,7 +229,7 @@ def p2l(obj, depth): if isinstance(obj, dict): return lua.table_from({ p2l(key, depth-1): p2l(value, depth-1) - for key, value in six.iteritems(obj) + for key, value in obj.items() }) if isinstance(obj, tuple) and keep_tuples: @@ -240,7 +239,7 @@ def p2l(obj, depth): tbl = lua.table_from([p2l(el, depth-1) for el in obj]) return _mark_table_as_array(lua, tbl) - if isinstance(obj, six.text_type): + if isinstance(obj, str): return obj.encode(encoding) if isinstance(obj, datetime.datetime): diff --git a/splash/lua_runner.py b/splash/lua_runner.py index 169ff356b..f11adcd9c 100644 --- a/splash/lua_runner.py +++ b/splash/lua_runner.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import abc import itertools -import six import lupa @@ -35,7 +33,7 @@ def yield_result(self, *args): self.dispatcher.dispatch(self.id, PyResult.yield_(*args)) -class BaseScriptRunner(six.with_metaclass(abc.ABCMeta, object)): +class BaseScriptRunner(metaclass=abc.ABCMeta): """ An utility class for running Lua coroutines. """ diff --git a/splash/lua_runtime.py b/splash/lua_runtime.py index 9ed3cf543..02fa17a49 100644 --- a/splash/lua_runtime.py +++ b/splash/lua_runtime.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import os import weakref import contextlib diff --git a/splash/network_manager.py b/splash/network_manager.py index 6e6718706..bc6c09dd4 100644 --- a/splash/network_manager.py +++ b/splash/network_manager.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import base64 import itertools import functools diff --git a/splash/pool.py b/splash/pool.py index 08802c843..520831570 100644 --- a/splash/pool.py +++ b/splash/pool.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import from twisted.internet import defer from twisted.python import log diff --git a/splash/proxy.py b/splash/proxy.py index a7ed66f8a..87158220b 100644 --- a/splash/proxy.py +++ b/splash/proxy.py @@ -5,14 +5,12 @@ which proxies to use for a given request. QNetworkManager calls a proxy factory for each outgoing request. """ -from __future__ import absolute_import import re import os +from urllib.parse import urlparse +import configparser from PyQt5.QtNetwork import QNetworkProxy -import six -from six.moves.urllib.parse import urlparse -from six.moves import configparser from splash.render_options import RenderOptions from splash.qtutils import create_proxy, validate_proxy_type @@ -35,8 +33,8 @@ def __init__(self, blacklist=None, whitelist=None, proxy_list=None): self.proxy_list = proxy_list or [] def queryProxy(self, query=None, *args, **kwargs): - protocol = six.text_type(query.protocolTag()) - url = six.text_type(query.url().toString()) + protocol = str(query.protocolTag()) + url = str(query.url().toString()) if self.should_use_proxy_list(protocol, url): return self._get_custom_proxy_list() return self._get_default_proxy_list() diff --git a/splash/qtrender.py b/splash/qtrender.py index daf468e18..70e45a96e 100644 --- a/splash/qtrender.py +++ b/splash/qtrender.py @@ -1,9 +1,7 @@ -from __future__ import absolute_import import abc import json import functools import pprint -import six from splash import defaults from splash.browser_tab import BrowserTab @@ -20,7 +18,7 @@ def stop_on_error_wrapper(self, *args, **kwargs): return stop_on_error_wrapper -class RenderScript(six.with_metaclass(abc.ABCMeta, object)): +class RenderScript(metaclass=abc.ABCMeta): """ Interface that all render scripts must implement. """ diff --git a/splash/qtrender_image.py b/splash/qtrender_image.py index b7bc7fe88..9b444b1a5 100644 --- a/splash/qtrender_image.py +++ b/splash/qtrender_image.py @@ -4,7 +4,6 @@ from abc import ABCMeta, abstractmethod, abstractproperty from io import BytesIO from math import ceil, floor -import six from PIL import Image from PyQt5.QtCore import QBuffer, QPoint, QRect, QSize, Qt @@ -292,9 +291,9 @@ def _render_qwebpage_tiled(self, web_rect, render_rect, canvas_size): # which is not what we want. painter.setViewport(render_rect) # painter.setClipRect(web_rect) - for i in six.moves.range(tile_conf['horizontal_count']): + for i in range(tile_conf['horizontal_count']): left = i * tile_qimage.width() - for j in six.moves.range(tile_conf['vertical_count']): + for j in range(tile_conf['vertical_count']): top = j * tile_qimage.height() painter.setViewport(render_rect.translated(-left, -top)) self.logger.log("Rendering with viewport=%s" @@ -375,7 +374,7 @@ def log(self, *args, **kwargs): pass -class WrappedImage(six.with_metaclass(ABCMeta, object)): +class WrappedImage(metaclass=ABCMeta): """ Base interface for operations with images of rendered webpages. diff --git a/splash/qtrender_lua.py b/splash/qtrender_lua.py index f7b7a53dc..077dd58ac 100644 --- a/splash/qtrender_lua.py +++ b/splash/qtrender_lua.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function import json import base64 import functools @@ -8,8 +7,7 @@ import time import sys import weakref - -import six +from urllib.parse import urlencode import twisted from PyQt5.QtCore import QTimer @@ -659,7 +657,7 @@ def go(self, url, baseurl=None, headers=None, http_method="GET", body=None, # XXX: should it be binary or unicode? body = self.lua.lua2python(formdata, max_depth=3, encoding=None) if isinstance(body, dict): - body = six.moves.urllib.parse.urlencode(body) + body = urlencode(body) else: raise ScriptError({ "argument": "formdata", @@ -858,7 +856,7 @@ def http_post(self, url, headers=None, follow_redirects=True, body=None): if isinstance(body, BinaryCapsule): body = body.data - if body and not isinstance(body, (six.text_type, bytes)): + if body and not isinstance(body, (str, bytes)): raise ScriptError({ "argument": "body", "message": "body argument for splash:http_post() must be string" @@ -946,7 +944,7 @@ def send_text(self, text): @command() def set_content(self, data, mime_type=None, baseurl=None): - if isinstance(data, six.text_type): + if isinstance(data, str): data = data.encode('utf8') def success(): @@ -1010,7 +1008,7 @@ def add_cookie(self, name, value, path=None, domain=None, expires=None, @command() def set_result_content_type(self, content_type): - if not isinstance(content_type, six.string_types): + if not isinstance(content_type, str): raise ScriptError({ "argument": "content_type", "message": "splash:set_result_content_type() argument " @@ -1030,7 +1028,7 @@ def set_result_status_code(self, code): @command() def set_result_header(self, name, value): - if not all([isinstance(h, six.string_types) for h in [name, value]]): + if not all([isinstance(h, str) for h in [name, value]]): raise ScriptError({ "message": "splash:set_result_header() arguments " "must be strings", @@ -1049,7 +1047,7 @@ def set_result_header(self, name, value): @command() def set_user_agent(self, value): - if not isinstance(value, six.string_types): + if not isinstance(value, str): raise ScriptError({ "argument": "value", "message": "splash:set_user_agent() argument must be a string", diff --git a/splash/qtutils.py b/splash/qtutils.py index 4c25e3f59..af213a22c 100644 --- a/splash/qtutils.py +++ b/splash/qtutils.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- """ Utils for working with QWebKit objects. """ -from __future__ import absolute_import - import functools import itertools import re @@ -17,7 +15,6 @@ from PyQt5.QtWebKit import QWebSettings from PyQt5.QtWebKitWidgets import QWebFrame from twisted.python import log -import six from splash.utils import truncated, to_bytes @@ -152,7 +149,7 @@ def get_qt_app(): def qurl2ascii(url): """ Convert QUrl to ASCII text suitable for logging """ - url = six.text_type(url.toString()).encode('unicode-escape').decode('ascii') + url = str(url.toString()).encode('unicode-escape').decode('ascii') if url.lower().startswith('data:'): return truncated(url, 80, '...[data uri truncated]') return url diff --git a/splash/qwebpage.py b/splash/qwebpage.py index 9e0d90125..4ce059a37 100644 --- a/splash/qwebpage.py +++ b/splash/qwebpage.py @@ -1,12 +1,10 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import from collections import namedtuple import sip from PyQt5.QtWebKitWidgets import QWebPage, QWebView from PyQt5.QtCore import QByteArray from twisted.python import log -import six from splash.har_builder import HarBuilder @@ -136,8 +134,8 @@ def extension(self, extension, info=None, errorPage=None): self.error_info = RenderErrorInfo( domain, int(info.error), - six.text_type(info.errorString), - six.text_type(info.url.toString()) + str(info.errorString), + str(info.url.toString()) ) # XXX: this page currently goes nowhere diff --git a/splash/render_options.py b/splash/render_options.py index 8ad47dbdb..31e144ce7 100644 --- a/splash/render_options.py +++ b/splash/render_options.py @@ -1,10 +1,7 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import os import json -import six - from splash import defaults from splash.utils import to_bytes, path_join_secure from splash.exceptions import BadOption @@ -86,7 +83,7 @@ def load_cached_args(self, cache): for name, key in (load_args or {}).items(): self.data[name] = cache[key] - def get(self, name, default=_REQUIRED, type=six.text_type, range=None): + def get(self, name, default=_REQUIRED, type=str, range=None): value = self.data.get(name) if value is not None: if type is not None: @@ -230,7 +227,7 @@ def get_headers(self): if isinstance(headers, (list, tuple)): for el in headers: - string_only = all(isinstance(e, six.string_types) for e in el) + string_only = all(isinstance(e, str) for e in el) if not (isinstance(el, (list, tuple)) and len(el) == 2 and string_only): self.raise_error( argument='headers', @@ -245,7 +242,7 @@ def get_save_args(self): if save_args is None: return [] - if isinstance(save_args, six.text_type): + if isinstance(save_args, str): # comma-separated string save_args = save_args.split(',') @@ -257,7 +254,7 @@ def get_save_args(self): ) # JSON array - if not all(isinstance(a, six.text_type) for a in save_args): + if not all(isinstance(a, str) for a in save_args): self.raise_error( argument="save_args", description="'save_args' should be a list of strings", @@ -269,7 +266,7 @@ def get_load_args(self): if load_args is None: return {} - if isinstance(load_args, six.text_type): + if isinstance(load_args, str): try: load_args = dict( kv.split("=", 1) for kv in load_args.split(';') @@ -340,13 +337,13 @@ def get_allowed_domains(self): def get_allowed_content_types(self): content_types = self.get("allowed_content_types", default=['*']) - if isinstance(content_types, six.text_type): + if isinstance(content_types, str): content_types = list(filter(None, content_types.split(','))) return content_types def get_forbidden_content_types(self): content_types = self.get("forbidden_content_types", default=[]) - if isinstance(content_types, six.text_type): + if isinstance(content_types, str): content_types = list(filter(None, content_types.split(','))) return content_types diff --git a/splash/request_middleware.py b/splash/request_middleware.py index 9baa5cc29..0d2e5fefc 100644 --- a/splash/request_middleware.py +++ b/splash/request_middleware.py @@ -4,13 +4,11 @@ various conditions. They should be used with :class:`splash.network_manager.SplashQNetworkAccessManager`. """ -from __future__ import absolute_import import re import os +from urllib.parse import urlsplit from twisted.python import log -import six -from six.moves.urllib.parse import urlsplit from splash.qtutils import request_repr, drop_request, get_request_webframe @@ -27,7 +25,7 @@ def __init__(self, allow_subdomains=True, verbosity=0): def process(self, request, render_options, operation, data): allowed_domains = render_options.get_allowed_domains() host_re = self._get_host_regex(allowed_domains, self.allow_subdomains) - if not host_re.match(six.text_type(request.url().host())): + if not host_re.match(str(request.url().host())): if self.verbosity >= 2: msg = "Dropped offsite %s" % request_repr(request, operation) log.msg(msg, system='request_middleware') @@ -122,7 +120,7 @@ def process(self, request, render_options, operation, data): else: return request - url = six.text_type(request.url().toString()) + url = str(request.url().toString()) browser_url = self._get_browser_url(request) domain = urlsplit(browser_url).hostname or '' # XXX: here we're using domain of a parent frame @@ -147,7 +145,7 @@ def _get_browser_url(self, request): return "" # in case of iframes use URL from 'address bar', not iframe's URL main_frame = current_frame.page().mainFrame() - return six.text_type(main_frame.url().toString()) + return str(main_frame.url().toString()) class AdblockRulesRegistry(object): diff --git a/splash/resources.py b/splash/resources.py index d5f94a666..8e6aceb40 100644 --- a/splash/resources.py +++ b/splash/resources.py @@ -2,7 +2,6 @@ This module contains Splash twisted.web Resources (HTTP API endpoints exposed to the user). """ -from __future__ import absolute_import import os import gc import time @@ -14,7 +13,6 @@ from twisted.web.static import File from twisted.internet import reactor, defer from twisted.python import log -import six import splash from splash.argument_cache import ArgumentCache @@ -175,7 +173,7 @@ def _write_output(self, data, request, content_type=None): request.setHeader(name, value) return self._write_output(data, request, content_type) - if data is None or isinstance(data, (bool, six.integer_types, float)): + if data is None or isinstance(data, (bool, int, float)): return self._write_output(str(data), request, content_type) if isinstance(data, BinaryCapsule): diff --git a/splash/response_middleware.py b/splash/response_middleware.py index 1cef4a526..f7e476bfe 100644 --- a/splash/response_middleware.py +++ b/splash/response_middleware.py @@ -4,7 +4,6 @@ various conditions. They should be used with :class:`splash.network_manager.SplashQNetworkAccessManager`. """ -from __future__ import absolute_import from PyQt5.QtNetwork import QNetworkRequest from splash.qtutils import request_repr from twisted.python import log diff --git a/splash/server.py b/splash/server.py index 22f066c44..1b7cc6656 100644 --- a/splash/server.py +++ b/splash/server.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import import os import sys import optparse diff --git a/splash/tests/conftest.py b/splash/tests/conftest.py index e724da5d2..b07837713 100644 --- a/splash/tests/conftest.py +++ b/splash/tests/conftest.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import - import pytest + from .utils import MockServers, SplashServer diff --git a/splash/tests/mockserver.py b/splash/tests/mockserver.py index d9020570b..b710b4aa0 100755 --- a/splash/tests/mockserver.py +++ b/splash/tests/mockserver.py @@ -1,12 +1,11 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function import os import optparse import base64 import random from functools import wraps -from six.moves.urllib.parse import unquote +from urllib.parse import unquote from twisted.web.server import Site, NOT_DONE_YET from twisted.web.resource import Resource diff --git a/splash/tests/stress.py b/splash/tests/stress.py index 120b0961d..644995a07 100644 --- a/splash/tests/stress.py +++ b/splash/tests/stress.py @@ -1,13 +1,10 @@ -from __future__ import print_function - import sys, random, optparse, time, json from itertools import islice from threading import Thread from collections import Counter -import requests +from queue import Queue -import six -from six.moves.queue import Queue +import requests from .utils import SplashServer, MockServer @@ -31,7 +28,7 @@ def run(self): starttime = time.time() q, p = Queue(), Queue() - for _ in six.moves.range(self.concurrency): + for _ in range(self.concurrency): t = Thread(target=worker, args=(self.host, q, p, self.verbose)) t.daemon = True t.start() @@ -40,7 +37,7 @@ def run(self): q.join() outputs = [] - for _ in six.moves.range(self.requests): + for _ in range(self.requests): outputs.append(p.get()) elapsed = time.time() - starttime diff --git a/splash/tests/test_argument_caching.py b/splash/tests/test_argument_caching.py index 3b2e37f7e..423f415c2 100644 --- a/splash/tests/test_argument_caching.py +++ b/splash/tests/test_argument_caching.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import hashlib from .test_render import BaseRenderTest diff --git a/splash/tests/test_callback_proxy.py b/splash/tests/test_callback_proxy.py index 76cff3242..4a92ece9b 100644 --- a/splash/tests/test_callback_proxy.py +++ b/splash/tests/test_callback_proxy.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import unittest from splash.browser_tab import OneShotCallbackError, OneShotCallbackProxy diff --git a/splash/tests/test_client_disconnects.py b/splash/tests/test_client_disconnects.py index cc8f6c4ef..ffd1672e9 100644 --- a/splash/tests/test_client_disconnects.py +++ b/splash/tests/test_client_disconnects.py @@ -1,12 +1,10 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import - import os import random import unittest import time -import six -from six.moves.http_client import HTTPConnection +from http.client import HTTPConnection +from urllib.parse import urlencode import pytest lupa = pytest.importorskip("lupa") @@ -47,7 +45,7 @@ def open_http_connection(self, code, query=None, method='GET'): q = {"lua_source": self.CREATE_FILE + "\n" + code} q.update(query or {}) conn = HTTPConnection('localhost', self.splash_unrestricted.portnum) - conn.request(method, "/execute/?" + six.moves.urllib.parse.urlencode(q)) + conn.request(method, "/execute/?" + urlencode(q)) return conn def assertScriptStopped(self, script, min_time=0.1, max_time=1.5): diff --git a/splash/tests/test_completer.py b/splash/tests/test_completer.py index 5b75e2142..586e68c31 100644 --- a/splash/tests/test_completer.py +++ b/splash/tests/test_completer.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import functools import pytest diff --git a/splash/tests/test_execute.py b/splash/tests/test_execute.py index 085878a9e..dedf1e5d1 100644 --- a/splash/tests/test_execute.py +++ b/splash/tests/test_execute.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import - import base64 import unittest from io import BytesIO @@ -9,7 +7,6 @@ from PIL import Image import requests -import six import pytest lupa = pytest.importorskip("lupa") @@ -39,7 +36,7 @@ def assertScriptError(self, resp, subtype, message=None): err = self.assertJsonError(resp, 400, 'ScriptError') self.assertEqual(err['info']['type'], subtype) if message is not None: - self.assertRegexpMatches(err['info']['message'], message) + self.assertRegex(err['info']['message'], message) return err def assertErrorLineNumber(self, resp, line_number): @@ -1013,7 +1010,7 @@ def test_runjs_exception(self): err = resp.json()['err'] self.assertEqual(err['type'], ScriptError.JS_ERROR) self.assertEqual(err['js_error_type'], 'ReferenceError') - self.assertRegexpMatches(err['message'], "Can't find variable") + self.assertRegex(err['message'], "Can't find variable") self.assertEqual(err['splash_method'], 'runjs') @@ -1100,10 +1097,7 @@ def test_throw_pcall(self): self.assertStatusCode(resp, 200) data = resp.json() self.assertEqual(data["ok"], False) - if six.PY3: - self.assertIn("error during JS function call: 'ABC'", data[u"res"]) - else: - self.assertIn("error during JS function call: u'ABC'", data[u"res"]) + self.assertIn("error during JS function call: 'ABC'", data[u"res"]) def test_throw_error(self): resp = self.request_lua(""" @@ -1138,10 +1132,7 @@ def test_throw_error_pcall(self): self.assertStatusCode(resp, 200) data = resp.json() self.assertEqual(data["ok"], False) - if six.PY3: - self.assertIn("error during JS function call: 'Error: ABC'", data[u"res"]) - else: - self.assertIn("error during JS function call: u'Error: ABC'", data[u"res"]) + self.assertIn("error during JS function call: 'Error: ABC'", data[u"res"]) def test_js_syntax_error(self): resp = self.request_lua(""" @@ -1537,12 +1528,8 @@ def test_go_multiple(self): }) self.assertStatusCode(resp, 200) data = resp.json() - if six.PY3: - self.assertIn("{b'foo': [b'1']}", data['html_1']) - self.assertIn("{b'bar': [b'2']}", data['html_2']) - else: - self.assertIn("{'foo': ['1']}", data['html_1']) - self.assertIn("{'bar': ['2']}", data['html_2']) + self.assertIn("{b'foo': [b'1']}", data['html_1']) + self.assertIn("{b'bar': [b'2']}", data['html_2']) def test_go_404_then_good(self): resp = self.request_lua(""" @@ -1904,14 +1891,9 @@ def test_set_user_agent(self): self.assertNotIn("Mozilla", data["res2"]) self.assertNotIn("Mozilla", data["res3"]) - if six.PY3: - self.assertNotIn("b'user-agent': b'Foozilla'", data["res1"]) - self.assertIn("b'user-agent': b'Foozilla'", data["res2"]) - self.assertIn("b'user-agent': b'Foozilla'", data["res3"]) - else: - self.assertNotIn("'user-agent': 'Foozilla'", data["res1"]) - self.assertIn("'user-agent': 'Foozilla'", data["res2"]) - self.assertIn("'user-agent': 'Foozilla'", data["res3"]) + self.assertNotIn("b'user-agent': b'Foozilla'", data["res1"]) + self.assertIn("b'user-agent': b'Foozilla'", data["res2"]) + self.assertIn("b'user-agent': b'Foozilla'", data["res3"]) def test_set_user_agent_base_url(self): resp = self.request_lua(""" @@ -3138,57 +3120,52 @@ def test_viewport_size(self): def test_viewport_size_validation(self): cases = [ - ('()', 'set_viewport_size.* takes exactly 3 arguments', + ('()', 'set_viewport_size.* missing 2 required positional arguments:*'), - ('{}', 'set_viewport_size.* takes exactly 3 arguments', + ('{}', 'set_viewport_size.* missing 2 required positional arguments:*'), - ('(1)', 'set_viewport_size.* takes exactly 3 arguments', + ('(1)', 'set_viewport_size.* missing 1 required positional argument:*'), - ('{1}', 'set_viewport_size.* takes exactly 3 arguments', + ('{1}', 'set_viewport_size.* missing 1 required positional argument:*'), - ('(1, nil)', 'a number is required', None), - ('{1, nil}', 'set_viewport_size.* takes exactly 3 arguments', + ('(1, nil)', 'a number is required'), + ('{1, nil}', 'set_viewport_size.* missing 1 required positional argument:*'), - ('(nil, 1)', 'a number is required', None), - ('{nil, 1}', 'a number is required', None), - ('{width=1}', 'set_viewport_size.* takes exactly 3 arguments', + ('(nil, 1)', 'a number is required'), + ('{nil, 1}', 'a number is required'), + ('{width=1}', 'set_viewport_size.* missing 1 required positional argument:*'), - ('{width=1, nil}', 'set_viewport_size.* takes exactly 3 arguments', + ('{width=1, nil}', 'set_viewport_size.* missing 1 required positional argument:*'), - ('{nil, width=1}', 'set_viewport_size.* takes exactly 3 arguments', + ('{nil, width=1}', 'set_viewport_size.* missing 1 required positional argument:*'), - ('{height=1}', 'set_viewport_size.* takes exactly 3 arguments', + ('{height=1}', 'set_viewport_size.* missing 1 required positional argument:*'), - ('{height=1, nil}', 'set_viewport_size.* takes exactly 3 arguments', + ('{height=1, nil}', 'set_viewport_size.* missing 1 required positional argument:*'), - ('{nil, height=1}', 'set_viewport_size.* takes exactly 3 arguments', + ('{nil, height=1}', 'set_viewport_size.* missing 1 required positional argument:*'), - ('{100, width=200}', 'set_viewport_size.* got multiple values.*width', None), + ('{100, width=200}', 'set_viewport_size.* got multiple values.*width'), # This thing works. # ('{height=200, 100}', 'set_viewport_size.* got multiple values.*width'), - ('{100, "a"}', 'a number is required', None), - ('{100, {}}', 'a number is required', None), + ('{100, "a"}', 'a number is required'), + ('{100, {}}', 'a number is required'), - ('{100, -1}', 'Viewport is out of range', None), - ('{100, 0}', 'Viewport is out of range', None), - ('{100, 99999}', 'Viewport is out of range', None), - ('{1, -100}', 'Viewport is out of range', None), - ('{0, 100}', 'Viewport is out of range', None), - ('{99999, 100}', 'Viewport is out of range', None), + ('{100, -1}', 'Viewport is out of range'), + ('{100, 0}', 'Viewport is out of range'), + ('{100, 99999}', 'Viewport is out of range'), + ('{1, -100}', 'Viewport is out of range'), + ('{0, 100}', 'Viewport is out of range'), + ('{99999, 100}', 'Viewport is out of range'), ] def run_test(size_str): self.get_dims_after('splash:set_viewport_size%s' % size_str) - for size_str, errmsg_py2, errmsg_py3 in cases: - if not errmsg_py3: - errmsg_py3 = errmsg_py2 - if six.PY3: - self.assertRaisesRegexp(RuntimeError, errmsg_py3, run_test, size_str) - else: - self.assertRaisesRegexp(RuntimeError, errmsg_py2, run_test, size_str) + for size_str, err_msg in cases: + self.assertRaisesRegex(RuntimeError, err_msg, run_test, size_str) def test_viewport_full(self): w = int(defaults.VIEWPORT_SIZE.split('x')[0]) @@ -3261,12 +3238,12 @@ def test_set_viewport_size_changes_contents_size_immediately(self): def test_viewport_full_raises_error_if_fails_in_script(self): # XXX: for local resources loadFinished event generally arrives after # initialLayoutCompleted, so the error doesn't manifest itself. - self.assertRaisesRegexp(RuntimeError, "zyzzy", - self.get_dims_after, - """ - splash:go(splash.args.url) - splash:set_viewport_full() - """, url=self.mockurl('delay')) + self.assertRaisesRegex(RuntimeError, "zyzzy", + self.get_dims_after, + """ + splash:go(splash.args.url) + splash:set_viewport_full() + """, url=self.mockurl('delay')) class RenderRegionTest(BaseLuaRenderTest): diff --git a/splash/tests/test_execute_callbacks.py b/splash/tests/test_execute_callbacks.py index 62b052908..13b90fb3f 100644 --- a/splash/tests/test_execute_callbacks.py +++ b/splash/tests/test_execute_callbacks.py @@ -1,11 +1,9 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import base64 from io import BytesIO +from urllib.parse import urlencode from PIL import Image -import six -from six.moves.urllib.parse import urlencode import pytest from splash.exceptions import ScriptError @@ -215,12 +213,8 @@ def test_set_header(self): """, {'url': self.mockurl("getrequest")}) self.assertStatusCode(resp, 200) - if six.PY3: - self.assertIn("b'custom-header': b'some-val'", resp.text) - self.assertIn("b'user-agent': b'Fooozilla'", resp.text) - else: - self.assertIn("'custom-header': 'some-val'", resp.text) - self.assertIn("'user-agent': 'Fooozilla'", resp.text) + self.assertIn("b'custom-header': b'some-val'", resp.text) + self.assertIn("b'user-agent': b'Fooozilla'", resp.text) def test_bad_callback(self): for arg in '', '"foo"', '123': @@ -405,7 +399,7 @@ def test_other_response_attrs(self): } mocked_url = self.mockurl("set-header?" + urlencode(headers)) some_attrs = { - "url": (six.text_type, mocked_url), + "url": (str, mocked_url), "status": (int, 200), "info": (dict, {}), "ok": (bool, True), diff --git a/splash/tests/test_execute_element.py b/splash/tests/test_execute_element.py index 486928cd5..f5b880560 100644 --- a/splash/tests/test_execute_element.py +++ b/splash/tests/test_execute_element.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import - import base64 from io import BytesIO from PIL import Image @@ -1448,7 +1446,7 @@ def test_submit(self): """, {"url": self.mockurl("various-elements")}) self.assertStatusCode(resp, 200) - self.assertRegexpMatches(resp.text, '/submitted\?username=admin&password=pass123') + self.assertRegex(resp.text, '/submitted\?username=admin&password=pass123') def test_submit_not_form(self): resp = self.request_lua(""" diff --git a/splash/tests/test_execute_emulation.py b/splash/tests/test_execute_emulation.py index 92a82edd5..38fc26720 100644 --- a/splash/tests/test_execute_emulation.py +++ b/splash/tests/test_execute_emulation.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import unittest import pytest diff --git a/splash/tests/test_execute_libraries.py b/splash/tests/test_execute_libraries.py index 8b92d69fa..b42f7cead 100644 --- a/splash/tests/test_execute_libraries.py +++ b/splash/tests/test_execute_libraries.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import base64 import json diff --git a/splash/tests/test_execute_request_filters.py b/splash/tests/test_execute_request_filters.py index e9031b9c5..ffed5b1e9 100644 --- a/splash/tests/test_execute_request_filters.py +++ b/splash/tests/test_execute_request_filters.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import - from splash.tests.test_execute import BaseLuaRenderTest from splash.tests.test_request_filters import BaseFiltersTest diff --git a/splash/tests/test_har.py b/splash/tests/test_har.py index 07abf0db3..2b98345e3 100644 --- a/splash/tests/test_har.py +++ b/splash/tests/test_har.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import base64 import requests import warnings diff --git a/splash/tests/test_jsonpost.py b/splash/tests/test_jsonpost.py index 4b1e11adf..4e31980ad 100644 --- a/splash/tests/test_jsonpost.py +++ b/splash/tests/test_jsonpost.py @@ -1,10 +1,8 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import json +from urllib.parse import urlencode import requests -from six.moves.urllib import parse as urlparse -import six from . import test_render, test_har, test_request_filters, test_runjs @@ -12,7 +10,7 @@ class JsonPostRequestHandler(test_render.DirectRequestHandler): def request(self, query, endpoint=None, headers=None): - assert not isinstance(query, six.string_types) + assert not isinstance(query, str) endpoint = endpoint or self.endpoint url = "http://%s/%s" % (self.host, endpoint) data = json.dumps(query) @@ -86,14 +84,9 @@ def test_get_headers(self): for r in [r1, r2]: self.assertStatusCode(r, 200) - if six.PY3: - self.assertIn("b'x-custom-header1': b'some-val1'", r.text) - self.assertIn("b'custom-header2': b'some-val2'", r.text) - self.assertIn("b'user-agent': b'Mozilla'", r.text) - else: - self.assertIn("'x-custom-header1': 'some-val1'", r.text) - self.assertIn("'custom-header2': 'some-val2'", r.text) - self.assertIn("'user-agent': 'Mozilla'", r.text) + self.assertIn("b'x-custom-header1': b'some-val1'", r.text) + self.assertIn("b'custom-header2': b'some-val2'", r.text) + self.assertIn("b'user-agent': b'Mozilla'", r.text) # This is not a proxy request, so Splash shouldn't remove # "Connection" header. @@ -129,10 +122,7 @@ def test_get_user_agent(self): "headers": headers, }) self.assertStatusCode(r, 200) - if six.PY3: - self.assertIn("b'user-agent': b'Mozilla123'", r.text) - else: - self.assertIn("'user-agent': 'Mozilla123'", r.text) + self.assertIn("b'user-agent': b'Mozilla123'", r.text) def test_connection_user_agent(self): headers = { @@ -146,25 +136,19 @@ def test_connection_user_agent(self): self.assertStatusCode(r, 200) # this is not a proxy request - don't remove headers - if six.PY3: - self.assertIn("b'user-agent': b'Mozilla123'", r.text) - else: - self.assertIn("'user-agent': 'Mozilla123'", r.text) + self.assertIn("b'user-agent': b'Mozilla123'", r.text) self.assertIn("mozilla123", r.text.lower()) def test_user_agent_after_redirect(self): headers = {'User-Agent': 'Mozilla123'} - query = urlparse.urlencode({"url": self.mockurl("getrequest")}) + query = urlencode({"url": self.mockurl("getrequest")}) r = self.request({ "url": self.mockurl("jsredirect-to?%s" % query), "headers": headers, "wait": 0.1, }) self.assertStatusCode(r, 200) - if six.PY3: - self.assertIn("b'user-agent': b'Mozilla123'", r.text) - else: - self.assertIn("'user-agent': 'Mozilla123'", r.text) + self.assertIn("b'user-agent': b'Mozilla123'", r.text) def test_cookie(self): r = self.request({ @@ -179,7 +163,7 @@ def test_http_POST_request_from_splash(self): r = self.request({ "url": self.mockurl("postrequest"), - "body": six.moves.urllib.parse.urlencode(formbody), + "body": urlencode(formbody), "http_method": "POST" }) self.assertStatusCode(r, 200) @@ -192,7 +176,7 @@ def test_http_go_POST_missing_method(self): formbody = {"param1": "one", "param2": "two"} r = self.request({ "url": self.mockurl("postrequest"), - "body": six.moves.urllib.parse.urlencode(formbody), + "body": urlencode(formbody), "baseurl": "foo" }) self.assertStatusCode(r, 400) @@ -208,7 +192,7 @@ def test_bad_http_method(self): # def test_cookie_after_redirect(self): # headers = {'Cookie': 'foo=bar'} - # query = urlparse.urlencode({"url": self.mockurl("get-cookie?key=foo")}) + # query = urlencode({"url": self.mockurl("get-cookie?key=foo")}) # r = self.request({ # "url": self.mockurl("jsredirect-to?%s" % query), # "headers": headers, diff --git a/splash/tests/test_jupyter.py b/splash/tests/test_jupyter.py index 22855e7e5..588c694fc 100644 --- a/splash/tests/test_jupyter.py +++ b/splash/tests/test_jupyter.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import - import pytest jupyter_kernel_test = pytest.importorskip("jupyter_kernel_test") lupa = pytest.importorskip("lupa") diff --git a/splash/tests/test_lua_parser.py b/splash/tests/test_lua_parser.py index 257850651..d576c1133 100644 --- a/splash/tests/test_lua_parser.py +++ b/splash/tests/test_lua_parser.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- """ Unit tests for Lua parser internals """ -from __future__ import absolute_import import functools import pytest diff --git a/splash/tests/test_lua_utils.py b/splash/tests/test_lua_utils.py index 6bf1bba34..f1b8cddc3 100644 --- a/splash/tests/test_lua_utils.py +++ b/splash/tests/test_lua_utils.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import unittest import pytest lupa = pytest.importorskip("lupa") diff --git a/splash/tests/test_proxy.py b/splash/tests/test_proxy.py index 88293acdf..38e9f5278 100644 --- a/splash/tests/test_proxy.py +++ b/splash/tests/test_proxy.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import os import shutil import unittest diff --git a/splash/tests/test_redirects.py b/splash/tests/test_redirects.py index 0425c0c0a..6d7ff5984 100644 --- a/splash/tests/test_redirects.py +++ b/splash/tests/test_redirects.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import unittest -import six + from splash.tests.test_render import BaseRenderTest from splash.tests.utils import NON_EXISTING_RESOLVABLE @@ -11,10 +10,7 @@ class HttpRedirectTest(BaseRenderTest): def assertRedirectedResponse(self, resp, code): self.assertStatusCode(resp, 200) self.assertIn("GET request", resp.text) - if six.PY3: - self.assertIn("{b'http_code': [b'%s']}" % code, resp.text) - else: - self.assertIn("{'http_code': ['%s']}" % code, resp.text) + self.assertIn("{b'http_code': [b'%s']}" % code, resp.text) def assertHttpRedirectWorks(self, code): resp = self.request({"url": self.mockurl("http-redirect?code=%s" % code)}) diff --git a/splash/tests/test_render.py b/splash/tests/test_render.py index d51e1d644..7685c73fc 100644 --- a/splash/tests/test_render.py +++ b/splash/tests/test_render.py @@ -4,11 +4,11 @@ import base64 from functools import wraps from io import BytesIO +from urllib.parse import urlencode import pytest import requests from PIL import Image, ImageChops -from six.moves.urllib import parse as urlparse from splash import defaults from splash.qtutils import has_min_qt_version @@ -335,7 +335,7 @@ def test_js_cookies_are_not_shared(self): def assertCookiesPreserved(self, use_js): use_js = "true" if use_js else "" get_cookie_url = self.mockurl("get-cookie?key=foo") - q = urlparse.urlencode({ + q = urlencode({ "key": "foo", "value": "bar", "next": get_cookie_url, diff --git a/splash/tests/test_request_filters.py b/splash/tests/test_request_filters.py index a9ba17591..8fb4844db 100644 --- a/splash/tests/test_request_filters.py +++ b/splash/tests/test_request_filters.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import os import shutil diff --git a/splash/tests/test_response_tracking.py b/splash/tests/test_response_tracking.py index d07bbda47..5c371175c 100644 --- a/splash/tests/test_response_tracking.py +++ b/splash/tests/test_response_tracking.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import base64 import requests diff --git a/splash/tests/test_runjs.py b/splash/tests/test_runjs.py index 987a40742..e1aff1bdc 100644 --- a/splash/tests/test_runjs.py +++ b/splash/tests/test_runjs.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import requests from .test_render import BaseRenderTest from .utils import SplashServer diff --git a/splash/tests/test_ui.py b/splash/tests/test_ui.py index 1efc181c3..683f926d6 100644 --- a/splash/tests/test_ui.py +++ b/splash/tests/test_ui.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import from splash import lua from .test_render import BaseRenderTest diff --git a/splash/tests/test_utils.py b/splash/tests/test_utils.py index 49441e050..1e0abfb79 100644 --- a/splash/tests/test_utils.py +++ b/splash/tests/test_utils.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import unittest from splash.utils import to_bytes, to_unicode diff --git a/splash/utils.py b/splash/utils.py index d9a9311c2..adcec377d 100644 --- a/splash/utils.py +++ b/splash/utils.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import os import gc import sys @@ -12,7 +10,6 @@ import psutil from uuid import uuid1 -import six _REQUIRED = object() @@ -45,9 +42,9 @@ def default(self, o): def to_unicode(text, encoding=None, errors='strict'): """Return the unicode representation of a bytes object `text`. If `text` is already an unicode object, return it as-is.""" - if isinstance(text, six.text_type): + if isinstance(text, str): return text - if not isinstance(text, (bytes, six.text_type)): + if not isinstance(text, (bytes, str)): raise TypeError('to_unicode must receive a bytes, str or unicode ' 'object, got %s' % type(text).__name__) if encoding is None: @@ -60,8 +57,8 @@ def to_bytes(text, encoding=None, errors='strict'): is already a bytes object, return it as-is.""" if isinstance(text, bytes): return text - if not isinstance(text, six.string_types): - raise TypeError('to_bytes must receive a unicode, str or bytes ' + if not isinstance(text, str): + raise TypeError('to_bytes must receive a str or bytes ' 'object, got %s' % type(text).__name__) if encoding is None: encoding = 'utf-8' diff --git a/splash/xvfb.py b/splash/xvfb.py index c6daef6d2..c974df7a9 100644 --- a/splash/xvfb.py +++ b/splash/xvfb.py @@ -3,7 +3,6 @@ Module for starting Xvfb automatically if it is available. Uses xvfbwrapper Python package. """ -from __future__ import absolute_import import sys from contextlib import contextmanager from splash import defaults