Implement screenshot test

Adds a screentest fixture with an is_valid method:
* adds a --set-new-base-screenshot command line parameter for setting a baseline
* uses same image-diff testing as examples

Still TODO:
* incorporate screenshots and diff into pytest html report
* upload screenshots and diff into
* prevent --set-new-base-screenshot on Travis (unlikely but still good to check)

Future work:
* hook in to perceptualdiff's tolerance feature - on a per-test basis?

.gitignore

@@ -14,6 +14,8 @@
 .sw[nop]
 *.tmp
 *.coverage.*
+**/screenshots/current*
+**/screenshots/diff*
 
 # Compiled source #
 ###################

tests/conftest.py

@@ -5,6 +5,7 @@
 
 pytest_plugins = (
     "tests.examples.examples_report_plugin",
+    "tests.integration.integration_tests_plugin",
     "tests.plugins.bokeh_server",
     "tests.plugins.jupyter_notebook",
     "tests.plugins.phantomjs_screenshot",

tests/integration/conftest.py → tests/integration/integration_tests_plugin.py

@@ -2,6 +2,13 @@
 import pytest
 
 from bokeh.io import output_file
+from .screenshot import Screenshot
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--set-new-base-screenshot", dest="set_new_base_screenshot", action="store_true", default=False, help="Use to set a new screenshot for imagediff testing. Be sure to only set for the tests you want by usign the -k pytest option to select your test."
+    )
 
 
 @pytest.fixture
@@ -39,3 +46,12 @@ def capabilities(capabilities):
     capabilities["browserName"] = "firefox"
     capabilities["tunnel-identifier"] = os.environ.get("TRAVIS_JOB_NUMBER")
     return capabilities
+
+
+@pytest.fixture
+def screenshot(request):
+    if request.config.option.set_new_base_screenshot:
+        screenshot = Screenshot(request=request, set_new_base=True)
+    else:
+        screenshot = Screenshot(request=request, set_new_base=False)
+    return screenshot

tests/integration/interaction/test_hover.py

@@ -0,0 +1,40 @@
+from __future__ import absolute_import
+
+from bokeh.io import save
+from bokeh.models import HoverTool
+from bokeh.plotting import figure
+from selenium.webdriver.common.action_chains import ActionChains
+
+import pytest
+pytestmark = pytest.mark.integration
+
+HEIGHT = 600
+WIDTH = 600
+
+
+def hover_at_position(selenium, canvas, x, y):
+    actions = ActionChains(selenium)
+    actions.move_to_element_with_offset(canvas, x, y)
+    actions.perform()
+
+
+def test_hover_changes_color(output_file_url, selenium, screenshot):
+
+    # Make plot and add a taptool callback that generates an alert
+    plot = figure(height=HEIGHT, width=WIDTH, tools='')
+    rect = plot.rect(
+        x=[1, 2], y=[1, 1],
+        width=1, height=1,
+        fill_color='cyan', hover_fill_color='magenta',
+        line_color=None,   hover_line_color=None
+    )
+    plot.add_tools(HoverTool(tooltips=None, renderers=[rect]))
+
+    # Save the plot and start the test
+    save(plot)
+    selenium.get(output_file_url)
+
+    # Hover over plot and take screenshot
+    canvas = selenium.find_element_by_tag_name('canvas')
+    hover_at_position(selenium, canvas, WIDTH * 0.33, HEIGHT * 0.5)
+    assert screenshot.is_valid()

tests/integration/screenshot.py

@@ -0,0 +1,50 @@
+from tests.plugins.image_diff import process_image_diff
+
+
+class Screenshot(object):
+
+    def __init__(self, item=None, request=None, set_new_base=False):
+        if item:
+            self.driver = getattr(item, '_driver', None)
+            thing = item
+        if request:
+            self.driver = getattr(request.node, '_driver', None)
+            thing = request
+        assert self.driver is not None
+        self.base_screenshot_path = self.get_screenshot_path_root(thing, 'base')
+        self.current_screenshot_path = self.get_screenshot_path_root(thing, 'current')
+        self.diff_screenshot_path = self.get_screenshot_path_root(thing, 'diff')
+        self.set_new_base = set_new_base
+
+    @classmethod
+    def get_screenshot_path_root(cls, item, prefix):
+        # Get the path for the screenshot based on the test name
+        #
+        # item: Can be item or request
+
+        screenshot_dir = item.fspath.dirpath().join('screenshots')
+        screenshot_dir.ensure_dir()
+        test_file = item.fspath.basename.split('.py')[0]
+        test_name = item.function.__name__
+        screenshot_path_root = screenshot_dir.join(prefix + '__' + test_file + '__' + test_name + '.png')
+        return screenshot_path_root.strpath
+
+    def set_current_screenshot(self):
+        self.driver.get_screenshot_as_file(self.current_screenshot_path)
+
+    def set_base_screenshot(self):
+        self.driver.get_screenshot_as_file(self.base_screenshot_path)
+
+    def is_valid(self):
+        self.set_current_screenshot()
+        if self.set_new_base:
+            self.set_base_screenshot()
+        image_diff_result = process_image_diff(
+            self.diff_screenshot_path,
+            self.base_screenshot_path,
+            self.current_screenshot_path
+        )
+        if image_diff_result == 0:
+            return True
+        else:
+            return False