Merge branch 'master' into py3

Conflicts:
	.gitignore
	splash/browser_tab.py
	splash/qtrender_lua.py
	splash/render_options.py
	splash/tests/test_execute.py

.gitignore

@@ -12,6 +12,7 @@ _trial_temp
 .splash-cache*
 .coverage
 env*
+.cache/
 htmlcov/
 
 # IPython notebooks

docs/api.rst

@@ -156,6 +156,18 @@ headers : JSON array or object : optional
     "User-Agent" header is special: is is used for all outgoing requests,
     unlike other headers.
 
+.. _arg-body:
+
+body : string : optional
+    Body of HTTP POST request to be sent if method is POST.
+    Default ``content-type`` header for POST requests is ``application/x-www-form-urlencoded``.
+
+.. _arg-method:
+
+http_method : string : optional
+    HTTP method of outgoing Splash request. Default method is GET. Splash also
+    supports POST.
+
 
 Examples
 ~~~~~~~~

docs/scripting-ref.rst

@@ -20,7 +20,7 @@ splash:go
 Go to an URL. This is similar to entering an URL in a browser
 address bar, pressing Enter and waiting until page loads.
 
-**Signature:** ``ok, reason = splash:go{url, baseurl=nil, headers=nil}``
+**Signature:** ``ok, reason = splash:go{url, baseurl=nil, headers=nil, http_method="GET", body=nil, formdata=nil}``
 
 **Parameters:**
 
@@ -30,6 +30,11 @@ address bar, pressing Enter and waiting until page loads.
   loaded from ``baseurl``: relative resource paths will be relative
   to ``baseurl``, and the browser will think ``baseurl`` is in address bar;
 * headers - a Lua table with HTTP headers to add/replace in the initial request.
+* http_method - optional, string with HTTP method to use when visiting url,
+  defaults to GET, Splash also supports POST.
+* body - optional, string with body for POST request
+* formdata - Lua table that will be converted to urlencoded POST body and sent
+  with header ``content-type: application/x-www-form-urlencoded``
 
 **Returns:** ``ok, reason`` pair. If ``ok`` is nil then error happened during
 page load; ``reason`` provides an information about error type.
@@ -794,7 +799,7 @@ i.e. no earlier than some of the async functions is called.
 splash:http_get
 ---------------
 
-Send an HTTP request and return a response without loading
+Send an HTTP GET request and return a response without loading
 the result to the browser window.
 
 **Signature:** ``response = splash:http_get{url, headers=nil, follow_redirects=true}``
@@ -831,6 +836,53 @@ To load a webpage to the browser use :ref:`splash-go`.
 
 .. _HAR response: http://www.softwareishard.com/blog/har-12-spec/#response
 
+.. _splash-http-post:
+
+splash:http_post
+----------------
+
+Send an HTTP POST request and return a response without loading
+the result to the browser window.
+
+**Signature:** ``response = splash:http_post{url, headers=nil, follow_redirects=true, body=nil}``
+
+**Parameters:**
+
+* url - URL to load;
+* headers - a Lua table with HTTP headers to add/replace in the initial request;
+* follow_redirects - whether to follow HTTP redirects.
+* body - string with body of request, if you intend to send form submission,
+  body should be urlencoded.
+
+**Returns:** a Lua table with the response in `HAR response`_ format.
+
+**Async:** yes.
+
+Example of form submission:
+
+.. code-block:: lua
+
+    local reply = splash:http_post{url="http://example.com", body="user=Frank&password=hunter2"}
+    -- reply.content.text contains raw HTML data
+    -- reply.status contains HTTP status code, as a number
+    -- see HAR docs for more info
+
+Example of JSON POST request:
+
+.. code-block:: lua
+
+    local reply = splash:http_post{url="http://example.com/post", body='{"alfa": "beta"}',
+                                   headers={["content-type"]="application/json"}}
+
+
+In addition to all HAR fields the response contains "ok" flag which is true
+for successful responses and false when error happened.
+
+This method doesn't change the current page contents and URL.
+To load a webpage to the browser use :ref:`splash-go`.
+
+.. _HAR response: http://www.softwareishard.com/blog/har-12-spec/#response
+
 
 .. _splash-set-content:
 

splash/browser_tab.py

@@ -281,6 +281,9 @@ def go(self, url, callback, errback, baseurl=None, http_method='GET',
         """
         self.store_har_timing("_onStarted")
 
+        if body is not None:
+            body = to_bytes(body)
+
         if baseurl:
             # If baseurl is used, we download the page manually,
             # then set its contents to the QWebPage and let it
@@ -393,7 +396,7 @@ def _on_baseurl_request_finished(self, callback, errback, baseurl, url):
             self.logger.log("Error loading %s: %s" % (url, reply.errorString()), min_level=1)
 
     def _load_url_to_mainframe(self, url, http_method, body=None, headers=None):
-        request = self.http_client.request_obj(url, headers=headers)
+        request = self.http_client.request_obj(url, headers=headers, body=body)
         meth = OPERATION_QT_CONSTANTS[http_method]
         if body is None:  # PyQT doesn't support body=None
             self.web_page.mainFrame().load(request, meth)
@@ -531,6 +534,16 @@ def http_get(self, url, callback, headers=None, follow_redirects=True):
             follow_redirects=follow_redirects
         )
 
+    def http_post(self, url, callback, headers=None, follow_redirects=True, body=None):
+        if body is not None:
+            body = to_bytes(body)
+
+        self.http_client.post(url,
+                              callback=callback,
+                              headers=headers,
+                              follow_redirects=follow_redirects,
+                              body=body)
+
     def evaljs(self, js_source, handle_errors=True):
         """
         Run JS code in page context and return the result.
@@ -809,7 +822,7 @@ def set_user_agent(self, value):
         """ Set User-Agent header for future requests """
         self.web_page.custom_user_agent = value
 
-    def request_obj(self, url, headers=None):
+    def request_obj(self, url, headers=None, body=None):
         """ Return a QNetworkRequest object """
         request = QNetworkRequest()
         request.setUrl(QUrl(url))
@@ -818,6 +831,12 @@ def request_obj(self, url, headers=None):
         if headers is not None:
             self.web_page.skip_custom_headers = True
             self._set_request_headers(request, headers)
+
+        if body and not request.hasRawHeader(b"content-type"):
+            # there is POST body but no content-type
+            # QT will set this header, but it will complain so better to do this here
+            request.setRawHeader(b"content-type", b"application/x-www-form-urlencoded")
+
         return request
 
     def request(self, url, callback, method='GET', body=None,
@@ -840,19 +859,35 @@ def request(self, url, callback, method='GET', body=None,
     def get(self, url, callback, headers=None, follow_redirects=True):
         """ Send a GET HTTP request; call the callback with the reply. """
         cb = functools.partial(
-            self._on_get_finished,
+            self._return_reply,
             callback=callback,
             url=url,
         )
         self.request(url, cb, headers=headers, follow_redirects=follow_redirects)
 
+    def post(self, url, callback, headers=None, follow_redirects=True, body=None):
+        """ Send HTTP POST request;
+        """
+        cb = functools.partial(self._return_reply, callback=callback, url=url)
+        self.request(url, cb, headers=headers, follow_redirects=follow_redirects, body=body,
+                     method="POST")
+
     def _send_request(self, url, callback, method='GET', body=None,
                       headers=None):
         # XXX: The caller must ensure self._delete_reply is called in a callback.
-        if method != 'GET':
+        if method.upper() not in ["POST", "GET"]:
             raise NotImplementedError()
-        request = self.request_obj(url, headers=headers)
-        reply = self.network_manager.get(request)
+
+        if body is not None:
+            assert isinstance(body, bytes)
+
+        request = self.request_obj(url, headers=headers, body=body)
+
+        if method.upper() == "POST":
+            reply = self.network_manager.post(request, body)
+        else:
+            reply = self.network_manager.get(request)
+
         reply.finished.connect(callback)
         self._replies.add(reply)
         return reply
@@ -874,6 +909,11 @@ def _on_request_finished(self, callback, method, body, headers,
                 callback()
                 return
 
+            # handle redirects after POST request
+            if method.upper() == "POST":
+                method = "GET"
+                body = None
+
             redirect_url = reply.url().resolved(redirect_url)
             self.request(
                 url=redirect_url,
@@ -887,7 +927,7 @@ def _on_request_finished(self, callback, method, body, headers,
         finally:
             self._delete_reply(reply)
 
-    def _on_get_finished(self, callback, url):
+    def _return_reply(self, callback, url):
         reply = self.sender()
         callback(reply)