diff --git a/docs/source/customize.rst b/docs/source/customize.rst index bc4fdea6c..8ca410435 100755 --- a/docs/source/customize.rst +++ b/docs/source/customize.rst @@ -209,6 +209,107 @@ There is a Voilà template cookiecutter available to give you a running start. This cookiecutter contains some docker configuration for live reloading of your template changes to make development easier. Please refer to the `cookiecutter repo `_ for more information on how to use the Voilà template cookiecutter. +Accessing the tornado request (`prelaunch-hook`) +--------------------------------------------------- + +In certain custom setups when you need to access the tornado request object in order to check for authentication cookies, access details about the request headers, or modify the notebook before rendering. You can leverage the `prelaunch-hook`, which lets you inject a function to inspect the notebook and the request prior to executing them. + +.. warning:: + Because `prelaunch-hook` only runs after receiving a new request but before the notebook is executed, it is incompatible with + `preheated kernels`. + +Creating a hook function +************************** +The format of this hook should be: + +.. code-block:: python + + def hook(req: tornado.web.RequestHandler, + notebook: nbformat.NotebookNode, + cwd: str) -> Optional[nbformat.NotebookNode]: + +- The first argument will be a reference to the tornado `RequetHandler`, with which you can inspect parameters, headers, etc. +- The second argument will be the `NotebookNode`, which you can mutate to e.g. inject cells or make other notebook-level modifications. +- The last argument is the current working directory should you need to mutate anything on disk. +- The return value of your hook function can either be `None`, or a `NotebookNode`. + +Adding the hook function to Voilà +*********************************** +There are two ways to add the hook function to Voila: + +- Using the `voila.py` configuration file: + +Here is an example of the configuration file. This file needs to be placed in the directory where you start Voilà. + +.. code-block:: python + + def hook_function(req, notebook, cwd): + """Do your stuffs here""" + return notebook + + c.Voila.prelaunch_hook = hook_function + +- Start Voila from a python script: + +Here is an example of a custom `prelaunch-hook` to execute a notebook with `papermill`: + +.. code-block:: python + + def parameterize_with_papermill(req, notebook, cwd): + import tornado + + # Grab parameters + parameters = req.get_argument("parameters", {}) + + # try to convert to dict if not e.g. string/unicode + if not isinstance(parameters, dict): + try: + parameters = tornado.escape.json_decode(parameters) + except ValueError: + parameters = None + + # if passed and a dict, use papermill to inject parameters + if parameters and isinstance(parameters, dict): + from papermill.parameterize import parameterize_notebook + + # setup for papermill + # + # these two blocks are done + # to avoid triggering errors + # in papermill's notebook + # loading logic + for cell in notebook.cells: + if 'tags' not in cell.metadata: + cell.metadata.tags = [] + if "papermill" not in notebook.metadata: + notebook.metadata.papermill = {} + + # Parameterize with papermill + return parameterize_notebook(notebook, parameters) + +To add this hook to your `Voilà` application: + +.. code-block:: python + + from voila.app import Voila + from voila.config import VoilaConfiguration + + # customize config how you like + config = VoilaConfiguration() + + # create a voila instance + app = Voila() + + # set the config + app.voila_configuration = config + + # set the prelaunch hook + app.prelaunch_hook = parameterize_with_papermill + + # launch + app.start() + + Adding your own static files ============================ @@ -323,7 +424,12 @@ Preheated kernels ================== Since Voilà needs to start a new jupyter kernel and execute the requested notebook in this kernel for every connection, this would lead to a long waiting time before the widgets can be displayed in the browser. -To reduce this waiting time, especially for the heavy notebooks, users can activate the preheating kernel option of Voilà, this option will enable two features: +To reduce this waiting time, especially for the heavy notebooks, users can activate the preheating kernel option of Voilà. + +.. warning:: + Because preheated kernels are not executed on request, this feature is incompatible with the `prelaunch-hook` functionality. + +This option will enable two features: - A pool of kernels is started for each notebook and kept in standby, then the notebook is executed in every kernel of its pool. When a new client requests a kernel, the preheated kernel in this pool is used and another kernel is started asynchronously to refill the pool. - The HTML version of the notebook is rendered in each preheated kernel and stored, when a client connects to Voila, under some conditions, the cached HTML is served instead of re-rendering the notebook. @@ -392,6 +498,9 @@ Partially pre-render notebook To benefit the acceleration of preheating kernel mode, the notebooks need to be pre-rendered before users actually connect to Voilà. But in many real-world cases, the notebook requires some user-specific data to render correctly the widgets, which makes pre-rendering impossible. To overcome this limit, Voilà offers a feature to treat the most used method for providing user data: the URL `query string`. +.. note:: + For more advanced interaction with the tornado request object, see the `prelaunch-hook` feature. + In normal mode, Voilà users can get the `query string` at run time through the ``QUERY_STRING`` environment variable: .. code-block:: python diff --git a/setup.cfg b/setup.cfg index 017c6e037..911bca5eb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -56,6 +56,7 @@ test = pytest pytest-rerunfailures pytest-tornasync + papermill visual_test = jupyterlab~=3.0 diff --git a/tests/app/prelaunch_hook_papermill_test.py b/tests/app/prelaunch_hook_papermill_test.py new file mode 100644 index 000000000..e1375b81a --- /dev/null +++ b/tests/app/prelaunch_hook_papermill_test.py @@ -0,0 +1,62 @@ +# tests prelaunch hook config +import pytest + +import os + +from urllib.parse import quote_plus + +BASE_DIR = os.path.dirname(__file__) + + +@pytest.fixture +def voila_notebook(notebook_directory): + return os.path.join(notebook_directory, 'print_parameterized.ipynb') + + +@pytest.fixture +def voila_config(): + def parameterize_with_papermill(req, notebook, cwd): + import tornado + + # Grab parameters + parameters = req.get_argument("parameters", {}) + + # try to convert to dict if not e.g. string/unicode + if not isinstance(parameters, dict): + try: + parameters = tornado.escape.json_decode(parameters) + except ValueError: + parameters = None + + # if passed and a dict, use papermill to inject parameters + if parameters and isinstance(parameters, dict): + from papermill.parameterize import parameterize_notebook + + # setup for papermill + # + # these two blocks are done + # to avoid triggering errors + # in papermill's notebook + # loading logic + for cell in notebook.cells: + if 'tags' not in cell.metadata: + cell.metadata.tags = [] + if "papermill" not in notebook.metadata: + notebook.metadata.papermill = {} + + # Parameterize with papermill + return parameterize_notebook(notebook, parameters) + + def config(app): + app.prelaunch_hook = parameterize_with_papermill + + return config + + +async def test_prelaunch_hook_papermill(http_server_client, base_url): + url = base_url + '?parameters=' + quote_plus('{"name":"Parameterized_Variable"}') + response = await http_server_client.fetch(url) + assert response.code == 200 + html_text = response.body.decode('utf-8') + assert 'Hi Parameterized_Variable' in html_text + assert 'test_template.css' not in html_text, "test_template should not be the default" diff --git a/tests/app/prelaunch_hook_test.py b/tests/app/prelaunch_hook_test.py new file mode 100644 index 000000000..4129eff96 --- /dev/null +++ b/tests/app/prelaunch_hook_test.py @@ -0,0 +1,37 @@ +# tests prelaunch hook config +import pytest + +import os + +from nbformat import NotebookNode + +BASE_DIR = os.path.dirname(__file__) + + +@pytest.fixture +def voila_notebook(notebook_directory): + return os.path.join(notebook_directory, 'print.ipynb') + + +@pytest.fixture +def voila_config(): + def foo(req, notebook, cwd): + argument = req.get_argument("test") + notebook.cells.append(NotebookNode({ + "cell_type": "code", + "execution_count": 0, + "metadata": {}, + "outputs": [], + "source": f"print(\"Hi prelaunch hook {argument}!\")\n" + })) + + def config(app): + app.prelaunch_hook = foo + return config + + +async def test_prelaunch_hook(http_server_client, base_url): + response = await http_server_client.fetch(base_url + "?test=blerg", ) + assert response.code == 200 + assert 'Hi Voilà' in response.body.decode('utf-8') + assert 'Hi prelaunch hook blerg' in response.body.decode('utf-8') diff --git a/tests/notebooks/print_parameterized.ipynb b/tests/notebooks/print_parameterized.ipynb new file mode 100644 index 000000000..a5d9f2e94 --- /dev/null +++ b/tests/notebooks/print_parameterized.ipynb @@ -0,0 +1,47 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "name = 'Voila'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print('Hi ' + name + '!')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/voila/app.py b/voila/app.py index 7cae8f911..a01e408cb 100644 --- a/voila/app.py +++ b/voila/app.py @@ -35,7 +35,7 @@ from traitlets.config.application import Application from traitlets.config.loader import Config -from traitlets import Unicode, Integer, Bool, Dict, List, default +from traitlets import Unicode, Integer, Bool, Dict, List, Callable, default from jupyter_server.services.kernels.handlers import KernelHandler, ZMQChannelsHandler from jupyter_server.services.contents.largefilemanager import LargeFileManager @@ -122,19 +122,19 @@ class Voila(Application): ) ) aliases = { - 'port': 'Voila.port', - 'static': 'Voila.static_root', - 'strip_sources': 'VoilaConfiguration.strip_sources', 'autoreload': 'Voila.autoreload', - 'template': 'VoilaConfiguration.template', - 'theme': 'VoilaConfiguration.theme', 'base_url': 'Voila.base_url', + 'port': 'Voila.port', + 'static': 'Voila.static_root', 'server_url': 'Voila.server_url', + 'pool_size': 'VoilaConfiguration.default_pool_size', 'enable_nbextensions': 'VoilaConfiguration.enable_nbextensions', 'nbextensions_path': 'VoilaConfiguration.nbextensions_path', 'show_tracebacks': 'VoilaConfiguration.show_tracebacks', 'preheat_kernel': 'VoilaConfiguration.preheat_kernel', - 'pool_size': 'VoilaConfiguration.default_pool_size' + 'strip_sources': 'VoilaConfiguration.strip_sources', + 'template': 'VoilaConfiguration.template', + 'theme': 'VoilaConfiguration.theme' } classes = [ VoilaConfiguration, @@ -240,6 +240,30 @@ class Voila(Application): cannot be determined reliably by the Jupyter notebook server (proxified or containerized setups for example).""")) + prelaunch_hook = Callable( + default_value=None, + allow_none=True, + config=True, + help=_( + """A function that is called prior to the launch of a new kernel instance + when a user visits the voila webpage. Used for custom user authorization + or any other necessary pre-launch functions. + + Should be of the form: + + def hook(req: tornado.web.RequestHandler, + notebook: nbformat.NotebookNode, + cwd: str) + + Although most customizations can leverage templates, if you need access + to the request object (e.g. to inspect cookies for authentication), + or to modify the notebook itself (e.g. to inject some custom structure, + althought much of this can be done by interacting with the kernel + in javascript) the prelaunch hook lets you do that. + """ + ), + ) + @property def display_url(self): if self.custom_display_url: @@ -426,6 +450,10 @@ def start(self): self.contents_manager = LargeFileManager(parent=self) preheat_kernel: bool = self.voila_configuration.preheat_kernel pool_size: int = self.voila_configuration.default_pool_size + + if preheat_kernel and self.prelaunch_hook: + raise Exception("`preheat_kernel` and `prelaunch_hook` are incompatible") + kernel_manager_class = voila_kernel_manager_factory( self.voila_configuration.multi_kernel_manager_class, preheat_kernel, @@ -539,7 +567,8 @@ def start(self): 'notebook_path': os.path.relpath(self.notebook_path, self.root_dir), 'template_paths': self.template_paths, 'config': self.config, - 'voila_configuration': self.voila_configuration + 'voila_configuration': self.voila_configuration, + 'prelaunch_hook': self.prelaunch_hook } )) else: @@ -553,7 +582,8 @@ def start(self): { 'template_paths': self.template_paths, 'config': self.config, - 'voila_configuration': self.voila_configuration + 'voila_configuration': self.voila_configuration, + 'prelaunch_hook': self.prelaunch_hook }), ]) diff --git a/voila/handler.py b/voila/handler.py index fa60f1aa5..0c4d88a0a 100644 --- a/voila/handler.py +++ b/voila/handler.py @@ -65,6 +65,9 @@ def initialize(self, **kwargs): self.notebook_path = kwargs.pop('notebook_path', []) # should it be [] self.template_paths = kwargs.pop('template_paths', []) self.traitlet_config = kwargs.pop('config', None) + self.voila_configuration = kwargs['voila_configuration'] + self.prelaunch_hook = kwargs.get('prelaunch_hook', None) + # we want to avoid starting multiple kernels due to template mistakes self.kernel_started = False @@ -77,6 +80,7 @@ async def get_generator(self, path=None): ): # when we are in single notebook mode but have a path self.redirect_to_file(path) return + cwd = os.path.dirname(notebook_path) # Adding request uri to kernel env @@ -89,8 +93,10 @@ async def get_generator(self, path=None): request_info[ENV_VARIABLE.SERVER_SOFTWARE] = 'voila/{}'.format(__version__) request_info[ENV_VARIABLE.SERVER_PROTOCOL] = str(self.request.version) host, port = split_host_and_port(self.request.host.lower()) + request_info[ENV_VARIABLE.SERVER_PORT] = str(port) if port else '' request_info[ENV_VARIABLE.SERVER_NAME] = host + # Add HTTP Headers as env vars following rfc3875#section-4.1.18 if len(self.voila_configuration.http_header_envs) > 0: for header_name in self.request.headers: @@ -116,6 +122,7 @@ async def get_generator(self, path=None): # For server extenstion case. current_notebook_data = {} pool_size = 0 + # Check if the conditions for using pre-heated kernel are satisfied. if self.should_use_rendered_notebook( current_notebook_data, @@ -163,6 +170,7 @@ async def get_generator(self, path=None): return gen = NotebookRenderer( + request_handler=self, voila_configuration=self.voila_configuration, traitlet_config=self.traitlet_config, notebook_path=notebook_path, @@ -171,6 +179,7 @@ async def get_generator(self, path=None): contents_manager=self.contents_manager, base_url=self.base_url, kernel_spec_manager=self.kernel_spec_manager, + prelaunch_hook=self.prelaunch_hook, ) await gen.initialize(template=template_arg, theme=theme_arg) diff --git a/voila/notebook_renderer.py b/voila/notebook_renderer.py index b8e18e7b4..f00f75066 100644 --- a/voila/notebook_renderer.py +++ b/voila/notebook_renderer.py @@ -31,6 +31,7 @@ class NotebookRenderer(LoggingConfigurable): def __init__(self, **kwargs): super().__init__() + self.request_handler = kwargs.get('request_handler') self.root_dir = kwargs.get('root_dir', []) self.notebook_path = kwargs.get('notebook_path', []) # should it be [] self.template_paths = kwargs.get('template_paths', []) @@ -39,6 +40,7 @@ def __init__(self, **kwargs): self.config_manager = kwargs.get('config_manager') self.contents_manager = kwargs.get('contents_manager') self.kernel_spec_manager = kwargs.get('kernel_spec_manager') + self.prelaunch_hook = kwargs.get('prelaunch_hook') self.default_kernel_name = 'python3' self.base_url = kwargs.get('base_url') self.kernel_started = False @@ -69,6 +71,19 @@ async def initialize(self, **kwargs) -> None: self.cwd = os.path.dirname(notebook_path) + if self.prelaunch_hook: + # Allow for preprocessing the notebook. + # Can be used to add auth, do custom formatting/standardization + # of the notebook, raise exceptions, etc + # + # Necessary inside of the handler if you need + # to access the tornado request itself + returned_notebook = self.prelaunch_hook(self.request_handler, + notebook=self.notebook, + cwd=self.cwd) + if returned_notebook: + self.notebook = returned_notebook + _, basename = os.path.split(notebook_path) notebook_name = os.path.splitext(basename)[0]