BackgroundJobs.ipynb

{
 "metadata": {
  "name": "BackgroundJobs"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "orig_nbformat": 2,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "source": [
      "# Simple interactive bacgkround jobs with IPython\n",
      "\n",
      "We start by loading the `backgroundjobs` library and defining a few trivial functions to illustrate things with."
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from IPython.lib import backgroundjobs as bg\n",
      "\n",
      "import sys\n",
      "import time\n",
      "\n",
      "def sleepfunc(interval=2, *a, **kw):\n",
      "    args = dict(interval=interval,\n",
      "                args=a,\n",
      "                kwargs=kw)\n",
      "    time.sleep(interval)\n",
      "    return args\n",
      "\n",
      "def diefunc(interval=2, *a, **kw):\n",
      "    time.sleep(interval)\n",
      "    raise Exception(\"Dead job with interval %s\" % interval)\n",
      "\n",
      "def printfunc(interval=1, reps=5):\n",
      "    for n in range(reps):\n",
      "        time.sleep(interval)\n",
      "        print 'In the background...', n\n",
      "        sys.stdout.flush()\n",
      "    print 'All done!'\n",
      "    sys.stdout.flush()"
     ],
     "language": "python",
     "outputs": [],
     "prompt_number": 35
    },
    {
     "cell_type": "markdown",
     "source": [
      "Now, we can create a job manager (called simply `jobs`) and use it to submit new jobs.\n",
      "<br>\n",
      "Run the cell below and wait a few seconds for the whole thing to finish, until you see the \"All done!\" printout."
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "jobs = bg.BackgroundJobManager()\n",
      "\n",
      "# Start a few jobs, the first one will have ID # 0\n",
      "jobs.new(sleepfunc, 4)\n",
      "jobs.new(sleepfunc, kw={'reps':2})\n",
      "jobs.new('printfunc(1,3)')\n",
      "\n",
      "# This makes a couple of jobs which will die.  Let's keep a reference to\n",
      "# them for easier traceback reporting later\n",
      "diejob1 = jobs.new(diefunc, 1)\n",
      "diejob2 = jobs.new(diefunc, 2)"
     ],
     "language": "python",
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Starting job # 0 in a separate thread.\n",
        "Starting job # 2 in a separate thread.\n",
        "Starting job # 3 in a separate thread.\n",
        "Starting job # 4 in a separate thread.\n",
        "Starting job # 5 in a separate thread.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "In the background... 0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "In the background... 1\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "In the background... 2\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "All done!\n"
       ]
      }
     ],
     "prompt_number": 36
    },
    {
     "cell_type": "markdown",
     "source": [
      "You can check the status of your jobs at any time:"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "jobs.status()"
     ],
     "language": "python",
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Completed jobs:\n",
        "0 : &lt;function sleepfunc at 0x30e1578&gt;\n",
        "2 : &lt;function sleepfunc at 0x30e1578&gt;\n",
        "3 : printfunc(1,3)\n",
        "\n",
        "Dead jobs:\n",
        "4 : &lt;function diefunc at 0x304d488&gt;\n",
        "5 : &lt;function diefunc at 0x304d488&gt;\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 37
    },
    {
     "cell_type": "markdown",
     "source": [
      "For any completed job, you can get its result easily:"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "jobs[0].result\n",
      "j0 = jobs[0]\n",
      "j0.join?"
     ],
     "language": "python",
     "outputs": [],
     "prompt_number": 43
    },
    {
     "cell_type": "markdown",
     "source": [
      "You can get the traceback of any dead job.  Run the line\n",
      "below again interactively until it prints a traceback (check the status\n",
      "of the job):\n"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print \"Status of diejob1:\", diejob1.status\n",
      "diejob1.traceback()  # jobs.traceback(4) would also work here, with the job number"
     ],
     "language": "python",
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Status of diejob1: Dead (Exception), call jobs.traceback() for details\n",
        "<span class=\"ansired\">---------------------------------------------------------------------------</span>\n",
        "<span class=\"ansired\">Exception</span>                                 Traceback (most recent call last)\n",
        "<span class=\"ansigreen\">/home/fperez/usr/lib/python2.6/site-packages/IPython/lib/backgroundjobs.py</span> in <span class=\"ansicyan\">call</span><span class=\"ansiblue\">(self)</span>\n",
        "<span class=\"ansigreen\">    462</span> <span class=\"ansiyellow\"></span>\n",
        "<span class=\"ansigreen\">    463</span>     <span class=\"ansigreen\">def</span> call<span class=\"ansiyellow\">(</span>self<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n",
        "<span class=\"ansigreen\">--&gt; 464</span><span class=\"ansiyellow\">         </span><span class=\"ansigreen\">return</span> self<span class=\"ansiyellow\">.</span>func<span class=\"ansiyellow\">(</span><span class=\"ansiyellow\">*</span>self<span class=\"ansiyellow\">.</span>args<span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">**</span>self<span class=\"ansiyellow\">.</span>kwargs<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n",
        "\n",
        "<span class=\"ansigreen\">/home/fperez/ipython/ipython/docs/examples/lib/&lt;ipython-input-15-54795a097787&gt;</span> in <span class=\"ansicyan\">diefunc</span><span class=\"ansiblue\">(interval, *a, **kw)</span>\n",
        "<span class=\"ansigreen\">     14</span> <span class=\"ansigreen\">def</span> diefunc<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">2</span><span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">*</span>a<span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">**</span>kw<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n",
        "<span class=\"ansigreen\">     15</span>     time<span class=\"ansiyellow\">.</span>sleep<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n",
        "<span class=\"ansigreen\">---&gt; 16</span><span class=\"ansiyellow\">     </span><span class=\"ansigreen\">raise</span> Exception<span class=\"ansiyellow\">(</span><span class=\"ansiblue\">&quot;Dead job with interval %s&quot;</span> <span class=\"ansiyellow\">%</span> interval<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n",
        "<span class=\"ansigreen\">     17</span> <span class=\"ansiyellow\"></span>\n",
        "<span class=\"ansigreen\">     18</span> <span class=\"ansigreen\">def</span> printfunc<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">1</span><span class=\"ansiyellow\">,</span> reps<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">5</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n",
        "\n",
        "<span class=\"ansired\">Exception</span>: Dead job with interval 1\n"
       ]
      }
     ],
     "prompt_number": 34
    },
    {
     "cell_type": "markdown",
     "source": [
      "This will print all tracebacks for all dead jobs:"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "jobs.traceback()"
     ],
     "language": "python",
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Traceback for: &lt;BackgroundJob #4: &lt;function diefunc at 0x30df758&gt;&gt;\n",
        "<span class=\"ansired\">---------------------------------------------------------------------------</span>\n",
        "<span class=\"ansired\">Exception</span>                                 Traceback (most recent call last)\n",
        "<span class=\"ansigreen\">/home/fperez/usr/lib/python2.6/site-packages/IPython/lib/backgroundjobs.py</span> in <span class=\"ansicyan\">call</span><span class=\"ansiblue\">(self)</span>\n",
        "<span class=\"ansigreen\">    462</span> <span class=\"ansiyellow\"></span>\n",
        "<span class=\"ansigreen\">    463</span>     <span class=\"ansigreen\">def</span> call<span class=\"ansiyellow\">(</span>self<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n",
        "<span class=\"ansigreen\">--&gt; 464</span><span class=\"ansiyellow\">         </span><span class=\"ansigreen\">return</span> self<span class=\"ansiyellow\">.</span>func<span class=\"ansiyellow\">(</span><span class=\"ansiyellow\">*</span>self<span class=\"ansiyellow\">.</span>args<span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">**</span>self<span class=\"ansiyellow\">.</span>kwargs<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n",
        "\n",
        "<span class=\"ansigreen\">/home/fperez/ipython/ipython/docs/examples/lib/&lt;ipython-input-15-54795a097787&gt;</span> in <span class=\"ansicyan\">diefunc</span><span class=\"ansiblue\">(interval, *a, **kw)</span>\n",
        "<span class=\"ansigreen\">     14</span> <span class=\"ansigreen\">def</span> diefunc<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">2</span><span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">*</span>a<span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">**</span>kw<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n",
        "<span class=\"ansigreen\">     15</span>     time<span class=\"ansiyellow\">.</span>sleep<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n",
        "<span class=\"ansigreen\">---&gt; 16</span><span class=\"ansiyellow\">     </span><span class=\"ansigreen\">raise</span> Exception<span class=\"ansiyellow\">(</span><span class=\"ansiblue\">&quot;Dead job with interval %s&quot;</span> <span class=\"ansiyellow\">%</span> interval<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n",
        "<span class=\"ansigreen\">     17</span> <span class=\"ansiyellow\"></span>\n",
        "<span class=\"ansigreen\">     18</span> <span class=\"ansigreen\">def</span> printfunc<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">1</span><span class=\"ansiyellow\">,</span> reps<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">5</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n",
        "\n",
        "<span class=\"ansired\">Exception</span>: Dead job with interval 1\n",
        "\n",
        "Traceback for: &lt;BackgroundJob #5: &lt;function diefunc at 0x30df758&gt;&gt;\n",
        "<span class=\"ansired\">---------------------------------------------------------------------------</span>\n",
        "<span class=\"ansired\">Exception</span>                                 Traceback (most recent call last)\n",
        "<span class=\"ansigreen\">/home/fperez/usr/lib/python2.6/site-packages/IPython/lib/backgroundjobs.py</span> in <span class=\"ansicyan\">call</span><span class=\"ansiblue\">(self)</span>\n",
        "<span class=\"ansigreen\">    462</span> <span class=\"ansiyellow\"></span>\n",
        "<span class=\"ansigreen\">    463</span>     <span class=\"ansigreen\">def</span> call<span class=\"ansiyellow\">(</span>self<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n",
        "<span class=\"ansigreen\">--&gt; 464</span><span class=\"ansiyellow\">         </span><span class=\"ansigreen\">return</span> self<span class=\"ansiyellow\">.</span>func<span class=\"ansiyellow\">(</span><span class=\"ansiyellow\">*</span>self<span class=\"ansiyellow\">.</span>args<span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">**</span>self<span class=\"ansiyellow\">.</span>kwargs<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n",
        "\n",
        "<span class=\"ansigreen\">/home/fperez/ipython/ipython/docs/examples/lib/&lt;ipython-input-15-54795a097787&gt;</span> in <span class=\"ansicyan\">diefunc</span><span class=\"ansiblue\">(interval, *a, **kw)</span>\n",
        "<span class=\"ansigreen\">     14</span> <span class=\"ansigreen\">def</span> diefunc<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">2</span><span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">*</span>a<span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">**</span>kw<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n",
        "<span class=\"ansigreen\">     15</span>     time<span class=\"ansiyellow\">.</span>sleep<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n",
        "<span class=\"ansigreen\">---&gt; 16</span><span class=\"ansiyellow\">     </span><span class=\"ansigreen\">raise</span> Exception<span class=\"ansiyellow\">(</span><span class=\"ansiblue\">&quot;Dead job with interval %s&quot;</span> <span class=\"ansiyellow\">%</span> interval<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n",
        "<span class=\"ansigreen\">     17</span> <span class=\"ansiyellow\"></span>\n",
        "<span class=\"ansigreen\">     18</span> <span class=\"ansigreen\">def</span> printfunc<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">1</span><span class=\"ansiyellow\">,</span> reps<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">5</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n",
        "\n",
        "<span class=\"ansired\">Exception</span>: Dead job with interval 2\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 33
    },
    {
     "cell_type": "markdown",
     "source": [
      "The job manager can be flushed of all completed jobs at any time:"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "jobs.flush()"
     ],
     "language": "python",
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "No jobs to flush.\n"
       ]
      }
     ],
     "prompt_number": 25
    },
    {
     "cell_type": "markdown",
     "source": [
      "After that, the status is simply empty:"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "jobs.status()"
     ],
     "language": "python",
     "outputs": [],
     "prompt_number": 27
    },
    {
     "cell_type": "markdown",
     "source": [
      "It's easy to wait on a job:"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "j = jobs.new(sleepfunc, 2)\n",
      "print \"Will wait for j now...\"\n",
      "sys.stdout.flush()\n",
      "j.join()\n",
      "print \"Result from j:\"\n",
      "j.result"
     ],
     "language": "python",
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Starting job # 7 in a separate thread.\n",
        "Will wait for j now...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Result from j:\n"
       ]
      },
      {
       "output_type": "pyout",
       "prompt_number": 46,
       "text": [
        "{&apos;args&apos;: (), &apos;interval&apos;: 2, &apos;kwargs&apos;: {}}"
       ]
      }
     ],
     "prompt_number": 46
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [],
     "language": "python",
     "outputs": []
    }
   ]
  }
 ]
}