fix mac installation issues

.gitignore

@@ -0,0 +1,10 @@
+# distutils/setuptools files
+build
+dist
+splash.egg-info
+
+# temp files & IDE settings
+_trial_temp
+*.pyc
+.idea
+.ipynb_checkpoints

debian/control

@@ -14,7 +14,8 @@ Depends: ${misc:Depends}, ${python:Depends},
  python-twisted,
  python-qt4,
  python-qt4reactor,
- python-raven
+ python-raven,
+ openssl
 Description: Page thumbnailing service
  This is the service that powers webpage thumbnailing for Autoscraping
  (templates and annotation runs)

requirements.txt

@@ -1,7 +1,9 @@
-PyQt
+# install PyQt4 and the following packages:
 twisted
 qt4reactor
+psutil
 
 # the following libraries are only required by tests
+pyOpenSSL
 requests
-PIL
+Pillow

splash/server.py

@@ -29,15 +29,26 @@ def start_logging(opts):
     else:
         logfile = sys.stderr
     log.startLogging(logfile)
-    log.msg("Open files limit: %d" % resource.getrlimit(resource.RLIMIT_NOFILE)[0])
 
 def splash_started(opts, stderr):
     if opts.logfile:
         stderr.write("Splash started - logging to: %s\n" % opts.logfile)
 
 def bump_nofile_limit():
-    _, n = resource.getrlimit(resource.RLIMIT_NOFILE)
-    resource.setrlimit(resource.RLIMIT_NOFILE, (n, n))
+    from twisted.python import log
+    log.msg("Open files limit: %d" % resource.getrlimit(resource.RLIMIT_NOFILE)[0])
+    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    values_to_try = [v for v in [hard, 100000, 10000] if v > soft]
+    for new_soft in values_to_try:
+        try:
+            resource.setrlimit(resource.RLIMIT_NOFILE, (new_soft, hard))
+        except ValueError:
+            continue
+        else:
+            log.msg("Open files limit increased from %d to %d" % (soft, new_soft))
+            break
+    else:
+        log.msg("Can't bump open files limit")
 
 def manhole_server():
     from twisted.internet import reactor
@@ -75,8 +86,8 @@ def main():
     install_qtreactor()
     opts, _ = parse_opts()
 
-    bump_nofile_limit()
     start_logging(opts)
+    bump_nofile_limit()
     monitor_maxrss(opts.maxrss)
     manhole_server()
     splash_server(opts.port)

splash/tests/test_render.py

@@ -3,6 +3,17 @@
 from PIL import Image
 from splash.tests.utils import TestServers
 
+class TestTestUtils(unittest.TestCase):
+
+    def test_mockserver_works(self):
+        r = requests.get('http://localhost:8998/jsrender')
+        self.assertEqual(r.status_code, 200)
+
+    def test_splashserver_works(self):
+        r = requests.get('http://localhost:8050/debug')
+        self.assertEqual(r.status_code, 200)
+
+
 class _RenderTest(unittest.TestCase):
 
     host = "localhost:8050"

splash/tests/utils.py

@@ -21,7 +21,11 @@ def __enter__(self):
             msg = "unable to start splash server. error code: %d - stderr follows: \n%s" % \
                 (self.proc.returncode, self.proc.stderr.read())
             raise RuntimeError(msg)
+
+        # wait until server starts writing debug messages,
+        # then wait a bit more to make it more likely to be online
         self.proc.stderr.readline()
+        time.sleep(0.2)
 
     def __exit__(self, exc_type, exc_value, traceback):
         self.proc.kill()
@@ -35,6 +39,7 @@ def __enter__(self):
         self.proc = Popen([sys.executable, '-u', '-m', 'splash.tests.mockserver'],
             stdout=PIPE, env=get_testenv())
         self.proc.stdout.readline()
+        time.sleep(0.1)
 
     def __exit__(self, exc_type, exc_value, traceback):
         self.proc.kill()

splash/utils.py

@@ -1,5 +1,6 @@
 import os, gc, inspect
 from collections import defaultdict
+import psutil
 
 _REQUIRED = object()
 
@@ -19,7 +20,8 @@ def getarg(request, name, default=_REQUIRED, type=str, range=None):
 
 PID = os.getpid()
 def get_num_fds():
-    return len(os.listdir("/proc/%s/fd" % PID))
+    proc = psutil.Process(PID)
+    return proc.get_num_fds()
 
 def get_leaks():
     relevant_types = frozenset(('SplashQWebPage', 'SplashQNetworkAccessManager',