big refactor to splash.qtrender (new splash.qtrender2 module) to mini…

…mize memory leaks. also closes #8

splash/qtrender2.py

@@ -0,0 +1,110 @@
+from PyQt4.QtWebKit import QWebPage, QWebSettings, QWebView
+from PyQt4.QtCore import Qt, QUrl, QBuffer, QSize
+from PyQt4.QtGui import QPainter, QImage
+from PyQt4.QtNetwork import QNetworkAccessManager, QNetworkRequest
+from twisted.internet import defer
+
+class RenderError(Exception):
+    pass
+
+class SplashQNetworkAccessManager(QNetworkAccessManager):
+
+    def __init__(self, *args, **kwargs):
+        super(SplashQNetworkAccessManager, self).__init__(*args, **kwargs)
+        self.sslErrors.connect(self._sslErrors)
+        self.finished.connect(self._finished)
+
+    def _sslErrors(self, reply, errors):
+        reply.ignoreSslErrors()
+
+    def _finished(self, reply):
+        reply.deleteLater()
+
+
+class SplashQWebPage(QWebPage):
+
+    def javaScriptAlert(self, frame, msg):
+        return
+
+    def javaScriptConfirm(self, frame, msg):
+        return False
+
+
+class HtmlRender(object):
+
+    def __init__(self, url, baseurl=None):
+        self.url = url
+        self.web_view = QWebView()
+        self.network_manager = SplashQNetworkAccessManager()
+        self.web_page = SplashQWebPage()
+        self.web_page.setNetworkAccessManager(self.network_manager)
+        self.web_view.setPage(self.web_page)
+        self.web_view.setAttribute(Qt.WA_DeleteOnClose, True)
+        settings = self.web_view.settings()
+        settings.setAttribute(QWebSettings.JavascriptEnabled, True)
+        settings.setAttribute(QWebSettings.PluginsEnabled, False)
+        settings.setAttribute(QWebSettings.PrivateBrowsingEnabled, True)
+        settings.setAttribute(QWebSettings.LocalStorageEnabled, True)
+
+        self.deferred = defer.Deferred()
+        request = QNetworkRequest()
+        request.setUrl(QUrl(url))
+        if baseurl:
+            self._baseUrl = QUrl(baseurl)
+            self.network_manager.finished.connect(self._requestFinished)
+            self.network_manager.get(request)
+        else:
+            self.web_page.loadFinished.connect(self._loadFinished)
+            self.web_page.mainFrame().load(request)
+
+    def close(self):
+        self.web_view.stop()
+        self.web_view.close()
+        self.web_page.deleteLater()
+        self.web_view.deleteLater()
+        self.network_manager.deleteLater()
+
+    def _requestFinished(self, reply):
+        self.web_page.networkAccessManager().finished.disconnect(self._requestFinished)
+        self.web_view.loadFinished.connect(self._loadFinished)
+        mimeType = reply.header(QNetworkRequest.ContentTypeHeader).toString()
+        self.web_view.page().mainFrame().setContent(reply.readAll(), mimeType, self._baseUrl)
+
+    def _loadFinished(self, ok):
+        if self.deferred.called:
+            return
+        if ok:
+            try:
+                self.deferred.callback(self._render())
+            except:
+                self.deferred.errback()
+        else:
+            self.deferred.errback(RenderError())
+
+    def _render(self):
+        frame = self.web_view.page().mainFrame()
+        return str(frame.toHtml().toUtf8())
+
+
+class PngRender(HtmlRender):
+
+    def __init__(self, url, baseurl=None, width=None, height=None, vwidth=1024, vheight=768):
+        HtmlRender.__init__(self, url, baseurl)
+        self.width = width
+        self.height = height
+        self.vwidth = vwidth
+        self.vheight = vheight
+
+    def _render(self):
+        self.web_page.setViewportSize(QSize(self.vwidth, self.vheight))
+        image = QImage(self.web_page.viewportSize(), QImage.Format_ARGB32)
+        painter = QPainter(image)
+        self.web_page.mainFrame().render(painter)
+        painter.end()
+        if self.width:
+            image = image.scaledToWidth(self.width, Qt.SmoothTransformation)
+        if self.height:
+            image = image.copy(0, 0, self.width, self.height)
+        b = QBuffer()
+        image.save(b, "png")
+        return str(b.data())

splash/resources.py

@@ -3,7 +3,7 @@
 from twisted.web.resource import Resource
 from twisted.internet import reactor, defer
 from twisted.python import log
-from splash.qtrender import HtmlRender, PngRender, RenderError
+from splash.qtrender2 import HtmlRender, PngRender, RenderError
 from splash.utils import getarg, BadRequest, get_num_fds, get_leaks
 from splash import sentry
 

splash/tests/mockserver.py

@@ -1,6 +1,7 @@
+import os
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.resource import Resource
-from twisted.internet import reactor
+from twisted.internet import reactor, ssl
 from twisted.internet.task import deferLater
 from splash.utils import getarg
 
@@ -135,12 +136,20 @@ def __init__(self):
         self.putChild("drop", Drop())
 
 
+def ssl_factory():
+    pem = os.path.join(os.path.dirname(__file__), "server.pem")
+    return ssl.DefaultOpenSSLContextFactory(pem, pem)
+
+
 if __name__ == "__main__":
     root = Root()
     factory = Site(root)
     port = reactor.listenTCP(8998, factory)
+    sslport = reactor.listenSSL(8999, factory, ssl_factory())
     def print_listening():
         h = port.getHost()
-        print "Mock server running at http://%s:%d" % (h.host, h.port)
+        s = sslport.getHost()
+        print "Mock server running at http://%s:%d & https://%s:%d" % \
+            (h.host, h.port, s.host, s.port)
     reactor.callWhenRunning(print_listening)
     reactor.run()

splash/tests/server.pem

@@ -0,0 +1,36 @@
+-----BEGIN CERTIFICATE-----
+MIIDBjCCAm+gAwIBAgIBATANBgkqhkiG9w0BAQQFADB7MQswCQYDVQQGEwJTRzER
+MA8GA1UEChMITTJDcnlwdG8xFDASBgNVBAsTC00yQ3J5cHRvIENBMSQwIgYDVQQD
+ExtNMkNyeXB0byBDZXJ0aWZpY2F0ZSBNYXN0ZXIxHTAbBgkqhkiG9w0BCQEWDm5n
+cHNAcG9zdDEuY29tMB4XDTAwMDkxMDA5NTEzMFoXDTAyMDkxMDA5NTEzMFowUzEL
+MAkGA1UEBhMCU0cxETAPBgNVBAoTCE0yQ3J5cHRvMRIwEAYDVQQDEwlsb2NhbGhv
+c3QxHTAbBgkqhkiG9w0BCQEWDm5ncHNAcG9zdDEuY29tMFwwDQYJKoZIhvcNAQEB
+BQADSwAwSAJBAKy+e3dulvXzV7zoTZWc5TzgApr8DmeQHTYC8ydfzH7EECe4R1Xh
+5kwIzOuuFfn178FBiS84gngaNcrFi0Z5fAkCAwEAAaOCAQQwggEAMAkGA1UdEwQC
+MAAwLAYJYIZIAYb4QgENBB8WHU9wZW5TU0wgR2VuZXJhdGVkIENlcnRpZmljYXRl
+MB0GA1UdDgQWBBTPhIKSvnsmYsBVNWjj0m3M2z0qVTCBpQYDVR0jBIGdMIGagBT7
+hyNp65w6kxXlxb8pUU/+7Sg4AaF/pH0wezELMAkGA1UEBhMCU0cxETAPBgNVBAoT
+CE0yQ3J5cHRvMRQwEgYDVQQLEwtNMkNyeXB0byBDQTEkMCIGA1UEAxMbTTJDcnlw
+dG8gQ2VydGlmaWNhdGUgTWFzdGVyMR0wGwYJKoZIhvcNAQkBFg5uZ3BzQHBvc3Qx
+LmNvbYIBADANBgkqhkiG9w0BAQQFAAOBgQA7/CqT6PoHycTdhEStWNZde7M/2Yc6
+BoJuVwnW8YxGO8Sn6UJ4FeffZNcYZddSDKosw8LtPOeWoK3JINjAk5jiPQ2cww++
+7QGG/g5NDjxFZNDJP1dGiLAxPW6JXwov4v0FmdzfLOZ01jDcgQQZqEpYlgpuI5JE
+WUQ9Ho4EzbYCOQ==
+-----END CERTIFICATE-----
+-----BEGIN RSA PRIVATE KEY-----
+MIIBPAIBAAJBAKy+e3dulvXzV7zoTZWc5TzgApr8DmeQHTYC8ydfzH7EECe4R1Xh
+5kwIzOuuFfn178FBiS84gngaNcrFi0Z5fAkCAwEAAQJBAIqm/bz4NA1H++Vx5Ewx
+OcKp3w19QSaZAwlGRtsUxrP7436QjnREM3Bm8ygU11BjkPVmtrKm6AayQfCHqJoT
+ZIECIQDW0BoMoL0HOYM/mrTLhaykYAVqgIeJsPjvkEhTFXWBuQIhAM3deFAvWNu4
+nklUQ37XsCT2c9tmNt1LAT+slG2JOTTRAiAuXDtC/m3NYVwyHfFm+zKHRzHkClk2
+HjubeEgjpj32AQIhAJqMGTaZVOwevTXvvHwNEH+vRWsAYU/gbx+OQB+7VOcBAiEA
+oolb6NMg/R3enNPvS1O4UU1H8wpaF77L4yiSWlE0p4w=
+-----END RSA PRIVATE KEY-----
+-----BEGIN CERTIFICATE REQUEST-----
+MIIBDTCBuAIBADBTMQswCQYDVQQGEwJTRzERMA8GA1UEChMITTJDcnlwdG8xEjAQ
+BgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3DQEJARYObmdwc0Bwb3N0MS5jb20w
+XDANBgkqhkiG9w0BAQEFAANLADBIAkEArL57d26W9fNXvOhNlZzlPOACmvwOZ5Ad
+NgLzJ1/MfsQQJ7hHVeHmTAjM664V+fXvwUGJLziCeBo1ysWLRnl8CQIDAQABoAAw
+DQYJKoZIhvcNAQEEBQADQQA7uqbrNTjVWpF6By5ZNPvhZ4YdFgkeXFVWi5ao/TaP
+Vq4BG021fJ9nlHRtr4rotpgHDX1rr+iWeHKsx4+5DRSy
+-----END CERTIFICATE REQUEST-----

splash/tests/test_render.py

@@ -42,7 +42,13 @@ class RenderHtmlTest(_RenderTest):
     render_format = "html"
 
     def test_ok(self):
-        r = self.request("url=http://localhost:8998/jsrender")
+        self._test_ok("http://localhost:8998/jsrender")
+
+    def test_ok_https(self):
+        self._test_ok("https://localhost:8999/jsrender")
+
+    def _test_ok(self, url):
+        r = self.request("url=%s" % url)
         self.assertEqual(r.status_code, 200)
         self.assertEqual(r.headers["content-type"].lower(), "text/html; charset=utf-8")
         self.assertTrue("Before" not in r.text)
@@ -63,7 +69,13 @@ class RenderPngTest(_RenderTest):
     render_format = "png"
 
     def test_ok(self):
-        r = self.request("url=http://localhost:8998/jsrender")
+        self._test_ok("http://localhost:8998/jsrender")
+
+    def test_ok_https(self):
+        self._test_ok("https://localhost:8999/jsrender")
+
+    def _test_ok(self, url):
+        r = self.request("url=%s" % url)
         self.assertEqual(r.status_code, 200)
         self.assertEqual(r.headers["content-type"], "image/png")
         img = Image.open(StringIO(r.content))

splash/utils.py

@@ -22,8 +22,9 @@ def get_num_fds():
     return len(os.listdir("/proc/%s/fd" % PID))
 
 def get_leaks():
-    relevant_types = frozenset(('CustomQWebPage', 'CustomQNetworkAccessManager',
-        'QWebView', 'HtmlRender', 'PngRender', 'QNetworkRequest'))
+    relevant_types = frozenset(('SplashQWebPage', 'SplashQNetworkAccessManager',
+        'QWebView', 'HtmlRender', 'PngRender', 'QNetworkRequest', 'QSize',
+        'QBuffer', 'QPainter', 'QImage'))
     leaks = defaultdict(int)
     gc.collect()
     for o in gc.get_objects():