GPT2 text generation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# References\n",
    "- How to get GPT2 345M: https://github.com/huggingface/pytorch-pretrained-BERT/issues/582\n",
    "- https://github.com/huggingface/pytorch-pretrained-BERT/blob/master/examples/run_gpt2.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "numpy: 1.16.2\n",
      "torch: 1.1.0\n",
      "pytorch_pretrained_bert: 0.6.2\n"
     ]
    }
   ],
   "source": [
    "from functools import partial\n",
    "from tqdm import trange\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "import pytorch_pretrained_bert\n",
    "from pytorch_pretrained_bert import GPT2LMHeadModel, GPT2Tokenizer\n",
    "for mod in (np, torch, pytorch_pretrained_bert):\n",
    "    print(f'{mod.__name__}: {mod.__version__}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build model GPT2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GPT2LMHeadModel(\n",
       "  (transformer): GPT2Model(\n",
       "    (wte): Embedding(50257, 1024)\n",
       "    (wpe): Embedding(1024, 1024)\n",
       "    (h): ModuleList(\n",
       "      (0): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (1): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (2): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (3): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (4): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (5): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (6): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (7): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (8): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (9): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (10): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (11): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (12): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (13): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (14): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (15): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (16): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (17): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (18): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (19): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (20): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (21): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (22): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (23): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (ln_f): BertLayerNorm()\n",
       "  )\n",
       "  (lm_head): GPT2LMHead(\n",
       "    (decoder): Linear(in_features=1024, out_features=50257, bias=False)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seed = 0\n",
    "np.random.seed(seed)\n",
    "torch.random.manual_seed(seed)\n",
    "torch.cuda.manual_seed(seed)\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "model_name_or_path = 'gpt2'\n",
    "model_name_or_path = 'torch_GPT2-345M/'\n",
    "enc = GPT2Tokenizer.from_pretrained('gpt2')\n",
    "\n",
    "model = GPT2LMHeadModel.from_pretrained(model_name_or_path)\n",
    "model.to(device)\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "line = \"Dummy\"\n",
    "tokens_tensor = torch.tensor(enc.encode(line)).reshape(1, -1)\n",
    "predictions, _ = model(tokens_tensor)\n",
    "vocab_size = predictions.shape[-1]\n",
    "assert vocab_size == 50257  # GPT2 vocab size is 50257"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([1, 2, 50257]), torch.Size([1, 2]))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions.shape, tokens_tensor.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Minimal example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[   34,   945,   547, 15646,   287]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_size=1\n",
    "context_tokens = enc.encode(prompt)\n",
    "context = torch.tensor(context_tokens, device=device, dtype=torch.long).unsqueeze(0).repeat(batch_size, 1)\n",
    "context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_k-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[   34,   945,   547, 15646,   287]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "context_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(198)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predicted_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Cars were invented in 1876.\\nThe car is the first car that was ever designed and manufactured'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prompt = \"Cars were invented in\"\n",
    "max_predictions = 16\n",
    "top_k = 2\n",
    "\n",
    "context_tokens = enc.encode(prompt)\n",
    "context_tensor = torch.tensor(context_tokens, dtype=torch.long).reshape(1, -1)\n",
    "\n",
    "prev = context_tensor\n",
    "output = context_tensor.clone()\n",
    "past = None\n",
    "for i in range(max_predictions):\n",
    "    #predictions, past = model(output, past=past)\n",
    "    predictions, past = model(prev, past=past)\n",
    "    context_size = prev.shape[1]\n",
    "    assert predictions.shape == (1, context_size, vocab_size)\n",
    "    \n",
    "    last_prediction = predictions[:, -1, :]\n",
    "    topk = torch.topk(last_prediction, 10)\n",
    "    prev = torch.tensor([[ topk.indices[0][top_k-1] ]])\n",
    "    output = torch.cat((output, prev), dim=-1)\n",
    "    \n",
    "enc.decode(output[0].tolist())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> **NOTE**: the first iteration `prev` is the context tensor and `past` is None. From the second iteration,\n",
    "> `prev` is just the last predicted token and all the history is stored in `past`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "GPT2 345M has 24 layers and d_model = 1024"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1024"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "64*16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(list, 24, torch.Size([1, 21]))"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(past), len(past), output.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[torch.Size([2, 1, 16, 20, 64]),\n",
       " torch.Size([2, 1, 16, 20, 64]),\n",
       " torch.Size([2, 1, 16, 20, 64]),\n",
       " torch.Size([2, 1, 16, 20, 64]),\n",
       " torch.Size([2, 1, 16, 20, 64]),\n",
       " torch.Size([2, 1, 16, 20, 64]),\n",
       " torch.Size([2, 1, 16, 20, 64]),\n",
       " torch.Size([2, 1, 16, 20, 64]),\n",
       " torch.Size([2, 1, 16, 20, 64]),\n",
       " torch.Size([2, 1, 16, 20, 64]),\n",
       " torch.Size([2, 1, 16, 20, 64]),\n",
       " torch.Size([2, 1, 16, 20, 64]),\n",
       " torch.Size([2, 1, 16, 20, 64]),\n",
       " torch.Size([2, 1, 16, 20, 64]),\n",
       " torch.Size([2, 1, 16, 20, 64]),\n",
       " torch.Size([2, 1, 16, 20, 64]),\n",
       " torch.Size([2, 1, 16, 20, 64]),\n",
       " torch.Size([2, 1, 16, 20, 64]),\n",
       " torch.Size([2, 1, 16, 20, 64]),\n",
       " torch.Size([2, 1, 16, 20, 64]),\n",
       " torch.Size([2, 1, 16, 20, 64]),\n",
       " torch.Size([2, 1, 16, 20, 64]),\n",
       " torch.Size([2, 1, 16, 20, 64]),\n",
       " torch.Size([2, 1, 16, 20, 64])]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[t.shape for t in past]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Minimal example with sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cars were invented in Britain and are owned by many people in Britain. The cars are used for driving\n"
     ]
    }
   ],
   "source": [
    "seed = 0\n",
    "prompt = \"Cars were invented in\"\n",
    "max_predictions = 16\n",
    "top_k = 40\n",
    "\n",
    "np.random.seed(seed)\n",
    "torch.random.manual_seed(seed)\n",
    "context_tokens = enc.encode(prompt)\n",
    "context_tensor = torch.tensor(context_tokens, dtype=torch.long).reshape(1, -1)\n",
    "\n",
    "prev = context_tensor\n",
    "output = context_tensor.clone()\n",
    "past = None\n",
    "for i in range(max_predictions):\n",
    "    #predictions, past = model(output, past=past)\n",
    "    predictions, past = model(prev, past=past)\n",
    "    context_size = prev.shape[1]\n",
    "    assert predictions.shape == (1, context_size, vocab_size)\n",
    "    \n",
    "    last_prediction = predictions[:, -1, :]\n",
    "    topk = torch.topk(last_prediction, 10)   \n",
    "    log_probs = F.softmax(topk.values, dim=-1)  # softmax among the top-k\n",
    "    rand_idx_in_topk = torch.multinomial(log_probs, num_samples=1)\n",
    "    predicted_index = topk.indices[0][rand_idx_in_topk]      \n",
    "    \n",
    "    prev = torch.tensor([[ predicted_index ]])\n",
    "    output = torch.cat((output, prev), dim=-1)\n",
    "    \n",
    "print(enc.decode(output[0].tolist()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Explore model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run -i textgen.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "decoder = partial(decoder_gpt2, enc=enc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "gen_text = partial(generate_text_gpt2, model, enc, decoder)\n",
    "#gen_text?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/10 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "context[<class 'list'>]: [2061, 466, 345, 760, 546, 10850, 18252, 290, 12068, 15417, 28403, 30]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [00:02<00:00,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 1 ========================================\n",
      " Share your thoughts below.<|endoftext|>\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "gen_text('What do you know about Machine Learning and Natural Language Processing?', \n",
    "         length=10, seed=0, sample=True, top_k=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/130 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "context[<class 'list'>]: [2061, 466, 345, 760, 546, 10850, 18252, 290, 12068, 15417, 28403, 30]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 130/130 [00:21<00:00,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 1 ========================================\n",
      "\n",
      "\n",
      "I am an expert in artificial intelligence and natural language processing.\n",
      "\n",
      "What is your background and expertise in Artificial Intelligence?\n",
      "\n",
      "I am a graduate from MIT Media Lab where I worked on the machine learning team.\n",
      "\n",
      "What is your favorite thing about artificial intelligence?\n",
      "\n",
      "My favorite thing about machine learning is that it's so easy. It's so easy to understand. It's a very intuitive thing to understand. Machine learning is really easy to understand as well. If you understand machine learning, you'll understand how to apply it for different tasks, and that makes it so easy to learn how to use it.\n",
      "\n",
      "How\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "gen_text('What do you know about Machine Learning and Natural Language Processing?', \n",
    "         length=130, seed=1, sample=True, top_k=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/130 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "context[<class 'list'>]: [2061, 466, 345, 760, 546, 10850, 18252, 290, 12068, 15417, 28403, 30]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 130/130 [00:23<00:00,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 1 ========================================\n",
      "\n",
      "\n",
      "I am an expert in artificial intelligence and natural language processing.\n",
      "\n",
      "What is your background and expertise in Artificial Intelligence?\n",
      "\n",
      "I am a graduate from MIT Media Lab where I worked on the machine learning team.\n",
      "\n",
      "What is your favorite thing about artificial intelligence?\n",
      "\n",
      "My favorite thing about machine learning is that it's so easy. It's so easy to understand. It's a very intuitive thing to understand. Machine learning is really easy to understand as well. If you understand machine learning, you'll understand how to apply it for different tasks, and that makes it so easy to learn how to use it.\n",
      "\n",
      "How\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "gen_text('What do you know about Machine Learning and Natural Language Processing?', \n",
    "         length=130, seed=1, sample=True, top_k=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/200 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "context[<class 'list'>]: [2061, 466, 345, 760, 546, 10850, 18252, 290, 12068, 15417, 28403, 30]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [00:35<00:00,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 1 ========================================\n",
      " Take a look at these links.\n",
      "\n",
      "What is Machine Learning and Natural Language Processing?\n",
      "\n",
      "Machine Learning is the process of gathering, processing, and understanding data. This can be applied to a wide range of fields of research. The term \"Natural Language Processing\" is used to describe the process by which computer programs are used to learn and recognize information. Machine Learning refers to the process of gathering, processing, and understanding data and can be applied to all areas of computer science. It is one of the most popular areas of research, with a large amount of data.\n",
      "\n",
      "Machine Learning, as well as its related applications such as Machine Learning-as-a-Service (MLA-as-a-Service) and Artificial Intelligence, are often considered \"technologies\" and thus have been treated in different ways. For the purpose of this article I am only discussing machine learning and AI in a technical context.\n",
      "\n",
      "Machine Learning is a process that is used to learn information by using\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "gen_text('What do you know about Machine Learning and Natural Language Processing?', \n",
    "         length=200, seed=2, sample=True, top_k=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/200 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "context[<class 'list'>]: [2061, 466, 345, 760, 546, 10850, 18252, 290, 12068, 15417, 28403, 30]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [00:35<00:00,  5.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 1 ========================================\n",
      " Tell us about it by hitting the comments below, and keep your eye out for updates.<|endoftext|>\n",
      "================================================================================\n"
     ]
    }
   ],
   "source": [
    "gen_text('What do you know about Machine Learning and Natural Language Processing?', \n",
    "         length=200, seed=2, sample=True, top_k=40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/200 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "context[<class 'list'>]: [2061, 318, 10850, 18252, 290, 12068, 15417, 28403, 30, 198, 198, 37573, 18252, 318]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [00:39<00:00,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What is Machine Learning and Natural Language Processing?\n",
      "\n",
      "Machine Learning is\n",
      "======================================== SAMPLE 1 ========================================\n",
      " a field from which humans make untapped skills like image recognition, speech synthesis, and criminal profiling. Once found, these skills are often applied in machine learning-in-situ (MLI), where human researchers attempt to image how information is being processed in different kinds of machines. This process can be quite an art, just to hone even the most basic skills involved like common sense.\n",
      "\n",
      "Natural language processing concerns algorithms, like natural speech recognition, who learn to understand language through words. It's a natural parallel with machine learning, which is how whether you describe running up a escalator or rustling feet prepared you for walking battering rams. It's not that at this level of effectiveness all scientists should be making their living as judges for nightmarish videos. Machine learning means increasing to millions of times faster and more accurate the image they're grasping. However, unsatisfied with doing justice to real life victims of technology horrors, boycotts, demands for transparency, regulations, bans,\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "gen_text('What is Machine Learning and Natural Language Processing?\\n\\nMachine Learning is', \n",
    "         length=200, seed=3, sample=True, top_p=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/150 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "context[<class 'list'>]: [2061, 318, 10850, 18252, 290, 12068, 15417, 28403, 30, 198, 198, 35364, 15417, 28403, 318, 257, 8478, 286, 10850, 18252, 220]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 150/150 [00:26<00:00,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What is Machine Learning and Natural Language Processing?\n",
      "\n",
      "Natural Language Processing is a branch of Machine Learning \n",
      "======================================== SAMPLE 1 ========================================\n",
      "̻ library. Machine Learning is the type of technology which uses multiple computational resources to teach artificial intelligence ※ a kind of program or software which shapes itself.\n",
      "\n",
      "In Machine Learning, one tries to create \"robots\" which perform many different tasks in order to learn new things. This way, we are using different fields, Neural Networks, Algorithms etcetera to create a \"machine\" which learns complex, difficult and rewarding tasks. A machine that can recognize pictures in a room and use its manual interpretation to accomplish the task, this is called Artificial Neural Network. ̻ Machine Learning is calculated using Deep Learning algorithms and one thing that has to be taken into account is that machine learning requires natural language processing. ̻\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "prompt = \"\"\"What is Machine Learning and Natural Language Processing?\n",
    "\n",
    "Natural Language Processing is a branch of Machine Learning \"\"\" \n",
    "gen_text(prompt,\n",
    "         length=150, seed=3, sample=True, top_p=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/150 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "context[<class 'list'>]: [2061, 318, 10850, 18252, 290, 12068, 15417, 28403, 30, 198, 198, 35364, 15417, 28403, 318, 257, 8478, 286, 10850, 18252, 220]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 150/150 [00:29<00:00,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What is Machine Learning and Natural Language Processing?\n",
      "\n",
      "Natural Language Processing is a branch of Machine Learning \n",
      "======================================== SAMPLE 1 ========================================\n",
      "̻that can produce machine-readable text such as documents, images or other data; processed it into machine-readable text, could then be processed again, a system of classification, tools and neural networks ̻to eventually produce image or other text to produce document ̻or any other data.\n",
      "\n",
      "There are a hierarchical categories of deep learning types processor labeled for various purposes,\n",
      "\n",
      "dClassifiers enable\n",
      "\n",
      "that saves energy and memory by working with Data sets that are very large, such as Texts .\n",
      "\n",
      ". Latent Classifiers enable the use of RNN based models in Learning tasks or Machine Learning tasks.\n",
      "\n",
      "See the following video to know more about Machine Learning for Texts:\n",
      "\n",
      "What are\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "prompt = \"\"\"What is Machine Learning and Natural Language Processing?\n",
    "\n",
    "Natural Language Processing is a branch of Machine Learning \"\"\" \n",
    "gen_text(prompt,\n",
    "         length=150, seed=3, sample=True, top_p=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GPT2LMHeadModel(\n",
       "  (transformer): GPT2Model(\n",
       "    (wte): Embedding(50257, 1024)\n",
       "    (wpe): Embedding(1024, 1024)\n",
       "    (h): ModuleList(\n",
       "      (0): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (1): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (2): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (3): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (4): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (5): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (6): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (7): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (8): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (9): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (10): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (11): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (12): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (13): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (14): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (15): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (16): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (17): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (18): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (19): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (20): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (21): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (22): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "      (23): Block(\n",
       "        (ln_1): BertLayerNorm()\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "        (ln_2): BertLayerNorm()\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (ln_f): BertLayerNorm()\n",
       "  )\n",
       "  (lm_head): GPT2LMHead(\n",
       "    (decoder): Linear(in_features=1024, out_features=50257, bias=False)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seed = 0\n",
    "np.random.seed(seed)\n",
    "torch.random.manual_seed(seed)\n",
    "torch.cuda.manual_seed(seed)\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "model_name_or_path = 'gpt2'\n",
    "model_name_or_path = 'torch_GPT2-345M/'\n",
    "enc = GPT2Tokenizer.from_pretrained('gpt2')\n",
    "\n",
    "model = GPT2LMHeadModel.from_pretrained(model_name_or_path)\n",
    "model.to(device)\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1024"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.config.n_ctx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125/125 [00:20<00:00,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 1 ========================================\n",
      "\n",
      "\n",
      "I'm a software engineer and I'm currently working on a project called \"Machine Learning for the Web\". I'm also a big fan of natural language processing and I'm working on a project called \"Natural Language Processing for the Web\".\n",
      "\n",
      "What's your favorite thing about machine learning?\n",
      "\n",
      "I love the fact that it's so easy to use. I love the fact that it's so easy to use.\n",
      "\n",
      "What's your favorite thing about natural language processing?\n",
      "\n",
      "I love the fact that it's so easy to use. I love the fact that it's so easy to use.\n",
      "\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "run_model('What do you know about Machine Learning and Natural Language Processing?', \n",
    "          length=125, seed=0, sample=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125/125 [00:24<00:00,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 1 ========================================\n",
      " Feel free to join us at the 2017 AI conference in San Francisco to learn more about it\n",
      "\n",
      "This article is part of The Digital Finance Podcast, where we'll be covering topics like AI and finance.\n",
      "\n",
      "What are you waiting for!? Listen now, and subscribe for more stuff like the episode is in this playlist:\n",
      "\n",
      "Subscribe to us on iTunes: iTunes, Stitcher, Apple Podcasts, Google Play, SoundCloud, Soundcloud Stitcher RSS, RSS: The digital finance podcast\n",
      "\n",
      "Check out other podcasts you might enjoy:<|endoftext|>\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "run_model('What do you know about Machine Learning and Natural Language Processing?', \n",
    "          length=125, seed=0, sample=True, top_k=40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125/125 [00:18<00:00,  7.25it/s]\n",
      "  0%|          | 0/125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 1 ========================================\n",
      " Skip all the introductory materials we make available (get up to speed by reading about these areas). Take a Google or TechNet course on Machine Learning. Learn how to use Coursera's MOOCs, MPI's, Dataslab OpenLab, and Velocity courses. Understands language learning, like etiquette and human language perception, across languages and cultural perspectives, is an important research area. Can you think of any obvious defense against this issue that involves a technical community? I ask because I suffer from community engineering paralysis. Thinking outside the box is hard, so I always try to take on projects and expertise in\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125/125 [00:16<00:00,  7.27it/s]\n",
      "  0%|          | 0/125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 2 ========================================\n",
      " Learn more about the power of Go's neural network.\n",
      "\n",
      "Go also has built-in support for text-based logic, thereby enabling applications that streamline complex algorithms by adding generic logic to plain text presentations.\n",
      "\n",
      "Wired eyes served me well in college\n",
      "\n",
      "In college, I used to cram the amount of books I read in two months into probably total books as many as 2000 in total. I acquired the ability to multitask and was able to analyze piles of data in very little time compared to other college students at the time and application developers in general.\n",
      "\n",
      "Now that I have relocated to Florida,\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125/125 [00:16<00:00,  7.02it/s]\n",
      "  0%|          | 0/125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 3 ========================================\n",
      " We are looking for junior engineers looking to get their foot in the door somewhere fascinating key installations engineering decision makers, software developers and data analysts...\n",
      "\n",
      "Vancouver's first high-performance Deep Learning CRUD engine Supercomputer M23 (DLR22-K0)\n",
      "\n",
      "18 May, 2017\n",
      "\n",
      "Ukrainian researchers Nana Atemi he have shown what might be the world's first multicenter deep learning implementation: the implementation of extensive machine learning algorithms on heavyweight academic commercial products...\n",
      "\n",
      "Canons 6.4 (Summer after Summer) - repository links for all the major improvements to Canonical GL Linus\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125/125 [00:18<00:00,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 4 ========================================\n",
      " Have a question? See our answers in the Answer Center. The Answers Plugin will ask you questions and ponder them over the course of a workweek. The only magic is your IQ, so wisdom speaks with your machine. Have a question or comment? Do you want to share your Watson Technologies Know-How(BTK) with the world? Use our new Facebook page. If you'd like to be investigated for breaking laws via cost and schedule, delete us from your Yahoo Groups. Or, connect with us on LinkedIn. I Pray, esp. Almighty God, an Eternity resting from everlasting glory. Amen. Frances:\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "run_model('What do you know about Machine Learning and Natural Language Processing?', \n",
    "          length=125, seed=0, sample=True, top_p=0.9, nsamples=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125/125 [00:18<00:00,  7.25it/s]\n",
      "  0%|          | 0/125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 1 ========================================\n",
      " Skip all the introductory materials we make available (get up to speed by reading about these areas). Take a Google or TechNet course on Machine Learning. Learn how to use Coursera's MOOCs, MPI's, Dataslab OpenLab, and Velocity courses. Understands language learning, like etiquette and human language perception, across languages and cultural perspectives, is an important research area. Can you think of any obvious defense against this issue that involves a technical community? I ask because I suffer from community engineering paralysis. Thinking outside the box is hard, so I always try to take on projects and expertise in\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125/125 [00:16<00:00,  7.27it/s]\n",
      "  0%|          | 0/125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 2 ========================================\n",
      " Learn more about the power of Go's neural network.\n",
      "\n",
      "Go also has built-in support for text-based logic, thereby enabling applications that streamline complex algorithms by adding generic logic to plain text presentations.\n",
      "\n",
      "Wired eyes served me well in college\n",
      "\n",
      "In college, I used to cram the amount of books I read in two months into probably total books as many as 2000 in total. I acquired the ability to multitask and was able to analyze piles of data in very little time compared to other college students at the time and application developers in general.\n",
      "\n",
      "Now that I have relocated to Florida,\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125/125 [00:16<00:00,  7.02it/s]\n",
      "  0%|          | 0/125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 3 ========================================\n",
      " We are looking for junior engineers looking to get their foot in the door somewhere fascinating key installations engineering decision makers, software developers and data analysts...\n",
      "\n",
      "Vancouver's first high-performance Deep Learning CRUD engine Supercomputer M23 (DLR22-K0)\n",
      "\n",
      "18 May, 2017\n",
      "\n",
      "Ukrainian researchers Nana Atemi he have shown what might be the world's first multicenter deep learning implementation: the implementation of extensive machine learning algorithms on heavyweight academic commercial products...\n",
      "\n",
      "Canons 6.4 (Summer after Summer) - repository links for all the major improvements to Canonical GL Linus\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125/125 [00:18<00:00,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 4 ========================================\n",
      " Have a question? See our answers in the Answer Center. The Answers Plugin will ask you questions and ponder them over the course of a workweek. The only magic is your IQ, so wisdom speaks with your machine. Have a question or comment? Do you want to share your Watson Technologies Know-How(BTK) with the world? Use our new Facebook page. If you'd like to be investigated for breaking laws via cost and schedule, delete us from your Yahoo Groups. Or, connect with us on LinkedIn. I Pray, esp. Almighty God, an Eternity resting from everlasting glory. Amen. Frances:\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "run_model('What do you know about Machine Learning and Natural Language Processing?', \n",
    "          length=125, seed=0, sample=True, top_p=0.9, nsamples=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125/125 [00:27<00:00,  4.50it/s]\n",
      "  0%|          | 0/125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 1 ========================================\n",
      " Answer our online quiz to learn more.\n",
      "\n",
      "All of NLP's basic functions are defined by the Schema library, which has been proven to work as a family of intuitive models of Bayesian statistics, evolved from R and coupled to Phi Machine Learning.\n",
      "\n",
      "GPio are applications, while machine learning is pioneered by the Google Brain and DeepMind AI teams.\n",
      "\n",
      "The last name of the author is \"Conrad\".\n",
      "\n",
      "\n",
      "This site uses cookies, if you would like to view the cookie details please visit www.cookiepolicy.org/party.php. There you will find all track of cookies,\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125/125 [00:21<00:00,  7.26it/s]\n",
      "  0%|          | 0/125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 2 ========================================\n",
      " Improving production-ready marketing is one thing, but if you're looking to achieve better image search results, you'll want to spend time learning how to use machine learning and deep learning techniques. Have you researched and have any question? Click below to leave a comment and get in touch!\n",
      "\n",
      "Image Credits: iStockphoto/romjones30<|endoftext|>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125/125 [00:17<00:00,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 3 ========================================\n",
      " H1N1Partnerships#4\n",
      "\n",
      "And now you hear laughter …<|endoftext|>\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "run_model('What do you know about Machine Learning and Natural Language Processing?', \n",
    "          length=125, seed=0, sample=True, temperature=0.9, nsamples=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:20<00:00,  7.31it/s]\n",
      "  0%|          | 0/500 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 1 ========================================\n",
      " Feel free to fill out our online survey and make some friends. If you want to know about Google's upcoming project and more. Also feel free to send us an email with questions you may have.\n",
      "\n",
      "\n",
      "Thanks for reading.<|endoftext|>\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 500/500 [01:26<00:00,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 1 ========================================\n",
      "\n",
      "\n",
      "No need to rush to register now, just keep reading!\n",
      "\n",
      "\n",
      "What are Machine Learning and Natural Language Processing?\n",
      "\n",
      "\n",
      "Many of the applications of machine learning and natural language processing are based on machine learning - a natural language approach for analysing objects or relationships that appear in large datasets in real time and which gives rise in this work to a number of fascinating applications and uses in areas as diverse as human-machine interaction systems, image editing to recognize natural human faces, speech synthesis for machine translation, web browsing and document creation, database management, real estate development, and so on and there is a lot more.\n",
      "\n",
      "\n",
      "There is much to write about but let me start with this simple point from another article:\n",
      "\n",
      "Linguistic analysis, or NLP (Natural language processing, or NLP as computers recognise spoken expressions instead of sentences) is a technology that combines machine analysis with the application of human natural language processing - what this usually means is that the computer has developed and adapted its own model to the nature of the things it is examining but at the same time this model learns from the environment as well. In some ways, linguistic algorithms are just speech or music recognition systems but at the same time linguistic applications are quite diverse, from natural language search and translation systems to machine learning and machine translation of text, to natural language editing. What has often caught the attention of both developers and researchers in artificial intelligence and natural languages computing is the possibility that it involves language knowledge and this research is a reflection of the way people work from today.\n",
      "\n",
      "\n",
      "Natural languages are written from within a particular culture or language. This language/culture is passed around through history and, although the details are not always clear, our own language is still what we speak every day, even some years later when we change our own cultures. If we speak French and our family member wants to order us a nice meal, the customer has spoken French, the restaurant knows what cuisine has been prepared by our own families since the last year and, since we don't yet learn French (or learn English well enough to comprehend French), the language we use and which words are used we use ourselves rather than the words we read on an academic paper.\n",
      "\n",
      "\n",
      "Natural languages are like languages: for most of us it is just a matter of remembering our everyday activities, such as shopping where shopping for clothes and food is the habit with which we learn. With the same kind of knowledge in hand (from watching television,\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "run_model('What do you know about Machine Learning and Natural Language Processing?', \n",
    "          length=100, seed=0, sample=True, top_k=40, temperature=1.2)\n",
    "run_model('What do you know about Machine Learning and Natural Language Processing?', \n",
    "          length=500, seed=None, sample=True, top_k=40, temperature=1.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:14<00:00,  6.79it/s]\n",
      "  0%|          | 0/100 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 1 ========================================\n",
      " Feel free to fill out our online survey and make some friends. If you want to know about Google's upcoming project and more. Also feel free to send us an email with questions you may have.\n",
      "\n",
      "\n",
      "Thanks for reading.<|endoftext|>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:14<00:00,  6.80it/s]\n",
      "  0%|          | 0/100 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 2 ========================================\n",
      "\n",
      "\n",
      "No need to rush to register now, just keep reading!\n",
      "\n",
      "\n",
      "What are Machine Learning and Natural Language Processing?\n",
      "\n",
      "\n",
      "Many of the applications of machine learning and natural language processing are based on machine learning - a natural language approach for analysing objects or relationships that appear in large datasets in real time and which gives rise in this work to a number of fascinating applications and uses in areas as diverse as human-machine interaction systems, image editing to recognize natural human faces, speech synthesis for machine\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:16<00:00,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 3 ========================================\n",
      " Post a comment below to share your deep insights in the field :)\n",
      "\n",
      "This post (Machine Learning/NNP/NLP Tutorial) is licensed under a Creative Commons Attribution 4.0 International License.<|endoftext|>\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "run_model('What do you know about Machine Learning and Natural Language Processing?', \n",
    "          length=100, seed=0, sample=True, top_k=40, temperature=1.2, nsamples=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [00:55<00:00,  5.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 1 ========================================\n",
      " Skip all the introductory materials we make available (get up to speed by reading about these areas). Take a Google or TechNet course on Machine Learning. Learn how to use Coursera's MOOCs, MPI's, Dataslab OpenLab, and Velocity courses. Understands language learning, like etiquette and human language perception, across languages and cultural perspectives, is an important research area. Can you think of any obvious defense against this issue that involves a technical community? I ask because I suffer from community engineering paralysis. Thinking outside the box is hard, so I always try to take on projects and expertise in fields that may be at distinct risk of extinction.\n",
      "\n",
      "Full Video Those are below and here. What are the keys to mtn zero's success? They're equity & vehicle sentiment categories. Automotive appraisal is a powerful part of property valuation. Our results showed that we can convert Motalic's Label of great detected vehicles group into a yield related sentiment metric around 90 percent. By ratios for empty space, lateral movement, and dorsal cross-section. We chose Morpho and Forseli as catalysts to convert the Monitorage quantometer reporting into an attractive sentiment risk indicator. They are also groundwork for Radixin, Ecomorphology –w/thegyro Barometer Advocante BEST TECH STORY Other Menu\n",
      "\n",
      "All statistics I supplied here are provided in connection with the data being processed by special functions within mtn zero. For some fields the corresponding\n",
      "================================================================================\n"
     ]
    }
   ],
   "source": [
    "run_model('What do you know about Machine Learning and Natural Language Processing?', \n",
    "          length=300, seed=0, sample=True, top_p=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [00:58<00:00,  2.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 1 ========================================\n",
      " Skip all the introductory materials we make available (get up to speed by reading about these areas). Take a Google or TechNet course on Machine Learning. Learn how to use Coursera's MOOCs, MPI's, Dataslab OpenLab, and Velocity courses. Understands language learning, like etiquette and human language perception, across languages and cultural perspectives, is an important research area. Can you think of any obvious defense against this issue that involves a technical community? I ask because I suffer from community engineering paralysis. Thinking outside the box is hard, so I always try to take on projects and expertise in fields that may be at distinct risk of extinction.\n",
      "\n",
      "Full Video Those are below and here. What are the keys to mtn zero's success? They're equity & vehicle sentiment categories. Automotive appraisal is a powerful part of property valuation. Our results showed that we can convert Motalic's Label of great detected vehicles group into a yield related sentiment metric around 90 percent. By ratios for empty space, lateral movement, and dorsal cross-section. We chose Morpho and Forseli as catalysts to convert the Monitorage quantometer reporting into an attractive sentiment risk indicator. They are also groundwork for Radixin, Ecomorphology –w/thegyro Barometer Advocante BEST TECH STORY Other Menu\n",
      "\n",
      "All statistics I supplied here are provided in connection with the data being processed by special functions within mtn zero. For some fields the corresponding\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "run_model('What do you know about Machine Learning and Natural Language Processing?', \n",
    "          length=300, seed=0, sample=True, top_p=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [01:03<00:00,  5.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 1 ========================================\n",
      " Skip all the introductory materials we make available (get up to speed by reading about these areas). Take a Google or TechNet course on Machine Learning. Learn how to use Coursera's MOOCs, MPI's, Dataslab OpenLab, and Velocity courses. Understands language learning, like etiquette and human language perception, across languages and cultural perspectives, is an important research area. Can you think of any obvious defense against this issue that involves a technical community? I ask because I suffer from community engineering paralysis. Thinking outside the box is hard, so I always try to take on projects and expertise in fields that may be at distinct risk of extinction.\n",
      "\n",
      "Full Video Those are below and here. What are the keys to mtn zero's success? They're equity & vehicle sentiment categories. Automotive appraisal is a powerful part of property valuation. Our results showed that we can convert Motalic's Label of great detected vehicles group into a yield related sentiment metric around 90 percent. By ratios for empty space, lateral movement, and dorsal cross-section. We chose Morpho and Forseli as catalysts to convert the Monitorage quantometer reporting into an attractive sentiment risk indicator. They are also groundwork for Radixin, Ecomorphology –w/thegyro Barometer Advocante BEST TECH STORY Other Menu\n",
      "\n",
      "All statistics I supplied here are provided in connection with the data being processed by special functions within mtn zero. For some fields the corresponding\n",
      "================================================================================\n"
     ]
    }
   ],
   "source": [
    "run_model('What do you know about Machine Learning and Natural Language Processing?', \n",
    "          length=300, seed=0, sample=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125/125 [00:19<00:00,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 1 ========================================\n",
      " Tell us your answers on Twitter.\n",
      "\n",
      "This article is part of an ongoing series of posts exploring the challenges of machine learning and natural language processing. Click here to see other posts on this topic.\n",
      "\n",
      "Machine Learning and Natural Language Processing<|endoftext|>\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "run_model('What do you know about Machine Learning and Natural Language Processing?', \n",
    "          length=125, seed=0, sample=True, top_k=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 512/512 [01:39<00:00,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROMPT: What do you know about Machine Learning and Natural Language Processing?\n",
      "======================================== SAMPLE 1 ========================================\n",
      "\n",
      "\n",
      "I am an expert in artificial intelligence and natural language processing.\n",
      "\n",
      "What is your background and expertise in Artificial Intelligence?\n",
      "\n",
      "I am a graduate from MIT Media Lab where I worked on the machine learning team.\n",
      "\n",
      "What is your favorite thing about artificial intelligence?\n",
      "\n",
      "My favorite thing about machine learning is that it's so easy. It's so easy to understand. It's a very intuitive thing to understand. Machine learning is really easy to understand as well. If you understand machine learning, you'll understand how to apply it for different tasks, and that makes it so easy to learn how to use it.\n",
      "\n",
      "How did you discover natural language processing (NLP) and how did you get into it?\n",
      "\n",
      "I discovered NLP when I was working with a group of kids at a high school, and the teachers would ask them to type out some sentences about what they wanted to do, and they would be able to understand it. They would write it down as a word list and we would then try to explain what that meant to them, and then they could do it on a computer screen and the computer would translate it.\n",
      "\n",
      "How did you get started with natural language processing?\n",
      "\n",
      "We were talking about it in the office, and one of the teachers said, \"Well, we should start with a language, or a language with a certain meaning, and see how we can apply it to natural language. So we started thinking of natural languages with meaning.\" I'm a linguist, so I knew it was a very important area for me to pursue. We started with some words, but the first step was to learn to write sentences that could be understood in a language that could actually communicate with us in that language.\n",
      "\n",
      "I'm sure there are a lot of people out there who have never heard of NLP, but I want to get the word out there and give you the idea of how it actually works. NLP can actually be very helpful to people with disabilities or those with sensory problems, and it's a great way to understand how they can do things without using words. It's very helpful to understand something without actually seeing it, and to see the words.\n",
      "\n",
      "How do you use NLP when you are trying to understand the meaning of words? How are you able to see what is being said without using words?\n",
      "\n",
      "It's a very difficult concept to grasp. We do not really have any language. We're talking about a collection of words, a collection\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "run_model('What do you know about Machine Learning and Natural Language Processing?', \n",
    "          length=512, seed=1, sample=True, top_k=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 128/128 [00:21<00:00,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What do you know about Machine Learning and Natural Language Processing?\n",
      "\n",
      "\n",
      "\n",
      "I'm a software engineer and I'm currently working on a project called \"Machine Learning for the Web\". I'm also a big fan of natural language processing and I'm working on a project called \"Natural Language Processing for the Web\".\n",
      "\n",
      "What's your favorite thing about machine learning?\n",
      "\n",
      "I love the fact that it's so easy to use. I love the fact that it's so easy to use.\n",
      "\n",
      "What's your favorite thing about natural language processing?\n",
      "\n",
      "I love the fact that it's so easy to use. I love the fact that it's so easy to use.\n",
      "\n",
      "What's\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "prompt = 'What do you know about Machine Learning and Natural Language Processing?'\n",
    "batch_size = 1\n",
    "length = 128\n",
    "temperature = 1\n",
    "top_k = 40\n",
    "sample = False\n",
    "seed = 0\n",
    "\n",
    "np.random.seed(seed)\n",
    "torch.random.manual_seed(seed)\n",
    "\n",
    "context = torch.tensor(enc.encode(prompt), device=device, dtype=torch.long).unsqueeze(0).repeat(batch_size, 1)\n",
    "prev = context\n",
    "output = context\n",
    "past = None\n",
    "with torch.no_grad():\n",
    "    for i in trange(length):\n",
    "        logits, past = model(prev, past=past)\n",
    "        logits = logits[:, -1, :] / temperature\n",
    "        logits = top_k_logits(logits, k=top_k)\n",
    "        log_probs = F.softmax(logits, dim=-1)\n",
    "        if sample:\n",
    "            prev = torch.multinomial(log_probs, num_samples=1)\n",
    "        else:\n",
    "            _, prev = torch.topk(logits, k=1, dim=-1)\n",
    "        output = torch.cat((output, prev), dim=1)\n",
    "\n",
    "out = output[:, context.shape[1]:].tolist()\n",
    "print(prompt + '\\n')\n",
    "for i in range(batch_size):\n",
    "    text = enc.decode(out[i])\n",
    "    print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "int"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(round(logits.shape[1] * 0.3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/128 [00:00<?, ?it/s]\u001b[A\n",
      "  1%|          | 1/128 [00:00<00:44,  2.85it/s]\u001b[A\n",
      "  2%|▏         | 2/128 [00:00<00:39,  3.20it/s]\u001b[A\n",
      "  2%|▏         | 3/128 [00:00<00:36,  3.45it/s]\u001b[A\n",
      "  3%|▎         | 4/128 [00:00<00:31,  3.94it/s]\u001b[A\n",
      "  4%|▍         | 5/128 [00:01<00:28,  4.24it/s]\u001b[A\n",
      "  5%|▍         | 6/128 [00:01<00:27,  4.51it/s]\u001b[A\n",
      "  5%|▌         | 7/128 [00:01<00:24,  4.85it/s]\u001b[A\n",
      "  6%|▋         | 8/128 [00:01<00:23,  5.03it/s]\u001b[A\n",
      "  7%|▋         | 9/128 [00:01<00:22,  5.21it/s]\u001b[A\n",
      "  8%|▊         | 10/128 [00:02<00:21,  5.43it/s]\u001b[A\n",
      "  9%|▊         | 11/128 [00:02<00:21,  5.51it/s]\u001b[A\n",
      "  9%|▉         | 12/128 [00:02<00:22,  5.13it/s]\u001b[A\n",
      " 10%|█         | 13/128 [00:02<00:22,  5.12it/s]\u001b[A\n",
      " 11%|█         | 14/128 [00:02<00:21,  5.25it/s]\u001b[A\n",
      " 12%|█▏        | 15/128 [00:03<00:21,  5.30it/s]\u001b[A\n",
      " 12%|█▎        | 16/128 [00:03<00:21,  5.31it/s]\u001b[A\n",
      " 13%|█▎        | 17/128 [00:03<00:20,  5.47it/s]\u001b[A\n",
      " 14%|█▍        | 18/128 [00:03<00:20,  5.48it/s]\u001b[A\n",
      " 15%|█▍        | 19/128 [00:04<00:28,  3.83it/s]\u001b[A\n",
      " 16%|█▌        | 20/128 [00:04<00:30,  3.51it/s]\u001b[A\n",
      " 16%|█▋        | 21/128 [00:04<00:28,  3.71it/s]\u001b[A\n",
      " 17%|█▋        | 22/128 [00:04<00:25,  4.22it/s]\u001b[A\n",
      " 18%|█▊        | 23/128 [00:04<00:23,  4.48it/s]\u001b[A\n",
      " 19%|█▉        | 24/128 [00:05<00:21,  4.91it/s]\u001b[A\n",
      " 20%|█▉        | 25/128 [00:05<00:19,  5.22it/s]\u001b[A\n",
      " 20%|██        | 26/128 [00:05<00:18,  5.45it/s]\u001b[A\n",
      " 21%|██        | 27/128 [00:05<00:17,  5.71it/s]\u001b[A\n",
      " 22%|██▏       | 28/128 [00:05<00:23,  4.27it/s]\u001b[A\n",
      " 23%|██▎       | 29/128 [00:06<00:29,  3.32it/s]\u001b[A\n",
      " 23%|██▎       | 30/128 [00:06<00:27,  3.57it/s]\u001b[A\n",
      " 24%|██▍       | 31/128 [00:06<00:24,  4.03it/s]\u001b[A\n",
      " 25%|██▌       | 32/128 [00:06<00:21,  4.43it/s]\u001b[A\n",
      " 26%|██▌       | 33/128 [00:07<00:20,  4.64it/s]\u001b[A\n",
      " 27%|██▋       | 34/128 [00:07<00:19,  4.94it/s]\u001b[A\n",
      " 27%|██▋       | 35/128 [00:07<00:17,  5.18it/s]\u001b[A\n",
      " 28%|██▊       | 36/128 [00:07<00:17,  5.33it/s]\u001b[A\n",
      " 29%|██▉       | 37/128 [00:07<00:17,  5.17it/s]\u001b[A\n",
      " 30%|██▉       | 38/128 [00:08<00:18,  4.81it/s]\u001b[A\n",
      " 30%|███       | 39/128 [00:08<00:17,  4.99it/s]\u001b[A\n",
      " 31%|███▏      | 40/128 [00:08<00:18,  4.76it/s]\u001b[A\n",
      " 32%|███▏      | 41/128 [00:08<00:19,  4.55it/s]\u001b[A\n",
      " 33%|███▎      | 42/128 [00:08<00:18,  4.78it/s]\u001b[A\n",
      " 34%|███▎      | 43/128 [00:09<00:17,  4.78it/s]\u001b[A\n",
      " 34%|███▍      | 44/128 [00:09<00:16,  5.10it/s]\u001b[A\n",
      " 35%|███▌      | 45/128 [00:09<00:15,  5.38it/s]\u001b[A\n",
      " 36%|███▌      | 46/128 [00:09<00:15,  5.19it/s]\u001b[A\n",
      " 37%|███▋      | 47/128 [00:09<00:14,  5.46it/s]\u001b[A\n",
      " 38%|███▊      | 48/128 [00:10<00:14,  5.62it/s]\u001b[A\n",
      " 38%|███▊      | 49/128 [00:10<00:15,  5.12it/s]\u001b[A\n",
      " 39%|███▉      | 50/128 [00:10<00:14,  5.33it/s]\u001b[A\n",
      " 40%|███▉      | 51/128 [00:10<00:14,  5.40it/s]\u001b[A\n",
      " 41%|████      | 52/128 [00:10<00:14,  5.37it/s]\u001b[A\n",
      " 41%|████▏     | 53/128 [00:10<00:13,  5.59it/s]\u001b[A\n",
      " 42%|████▏     | 54/128 [00:11<00:13,  5.64it/s]\u001b[A\n",
      " 43%|████▎     | 55/128 [00:11<00:13,  5.38it/s]\u001b[A\n",
      " 44%|████▍     | 56/128 [00:11<00:13,  5.29it/s]\u001b[A\n",
      " 45%|████▍     | 57/128 [00:11<00:13,  5.21it/s]\u001b[A\n",
      " 45%|████▌     | 58/128 [00:11<00:13,  5.16it/s]\u001b[A\n",
      " 46%|████▌     | 59/128 [00:12<00:13,  5.17it/s]\u001b[A\n",
      " 47%|████▋     | 60/128 [00:12<00:12,  5.35it/s]\u001b[A\n",
      " 48%|████▊     | 61/128 [00:12<00:15,  4.20it/s]\u001b[A\n",
      " 48%|████▊     | 62/128 [00:12<00:15,  4.24it/s]\u001b[A\n",
      " 49%|████▉     | 63/128 [00:13<00:14,  4.37it/s]\u001b[A\n",
      " 50%|█████     | 64/128 [00:13<00:13,  4.69it/s]\u001b[A\n",
      " 51%|█████     | 65/128 [00:13<00:14,  4.48it/s]\u001b[A\n",
      " 52%|█████▏    | 66/128 [00:13<00:13,  4.61it/s]\u001b[A\n",
      " 52%|█████▏    | 67/128 [00:13<00:13,  4.50it/s]\u001b[A\n",
      " 53%|█████▎    | 68/128 [00:14<00:12,  4.65it/s]\u001b[A\n",
      " 54%|█████▍    | 69/128 [00:14<00:15,  3.73it/s]\u001b[A\n",
      " 55%|█████▍    | 70/128 [00:15<00:18,  3.14it/s]\u001b[A\n",
      " 55%|█████▌    | 71/128 [00:15<00:18,  3.12it/s]\u001b[A\n",
      " 56%|█████▋    | 72/128 [00:15<00:15,  3.66it/s]\u001b[A\n",
      " 57%|█████▋    | 73/128 [00:15<00:13,  4.11it/s]\u001b[A\n",
      " 58%|█████▊    | 74/128 [00:15<00:13,  3.96it/s]\u001b[A\n",
      " 59%|█████▊    | 75/128 [00:16<00:16,  3.31it/s]\u001b[A\n",
      " 59%|█████▉    | 76/128 [00:16<00:13,  3.78it/s]\u001b[A\n",
      " 60%|██████    | 77/128 [00:16<00:11,  4.28it/s]\u001b[A\n",
      " 61%|██████    | 78/128 [00:16<00:10,  4.64it/s]\u001b[A\n",
      " 62%|██████▏   | 79/128 [00:17<00:10,  4.82it/s]\u001b[A\n",
      " 62%|██████▎   | 80/128 [00:17<00:12,  3.90it/s]\u001b[A\n",
      " 63%|██████▎   | 81/128 [00:17<00:14,  3.16it/s]\u001b[A\n",
      " 64%|██████▍   | 82/128 [00:18<00:16,  2.80it/s]\u001b[A\n",
      " 65%|██████▍   | 83/128 [00:18<00:17,  2.60it/s]\u001b[A\n",
      " 66%|██████▌   | 84/128 [00:19<00:16,  2.62it/s]\u001b[A\n",
      " 66%|██████▋   | 85/128 [00:19<00:15,  2.69it/s]\u001b[A\n",
      " 67%|██████▋   | 86/128 [00:19<00:15,  2.68it/s]\u001b[A\n",
      " 68%|██████▊   | 87/128 [00:20<00:14,  2.92it/s]\u001b[A\n",
      " 69%|██████▉   | 88/128 [00:20<00:11,  3.36it/s]\u001b[A\n",
      " 70%|██████▉   | 89/128 [00:20<00:10,  3.74it/s]\u001b[A\n",
      " 70%|███████   | 90/128 [00:20<00:09,  4.20it/s]\u001b[A\n",
      " 71%|███████   | 91/128 [00:20<00:08,  4.48it/s]\u001b[A\n",
      " 72%|███████▏  | 92/128 [00:21<00:07,  4.72it/s]\u001b[A\n",
      " 73%|███████▎  | 93/128 [00:21<00:07,  4.85it/s]\u001b[A\n",
      " 73%|███████▎  | 94/128 [00:21<00:06,  5.00it/s]\u001b[A\n",
      " 74%|███████▍  | 95/128 [00:21<00:07,  4.71it/s]\u001b[A\n",
      " 75%|███████▌  | 96/128 [00:21<00:06,  4.88it/s]\u001b[A\n",
      " 76%|███████▌  | 97/128 [00:22<00:06,  4.68it/s]\u001b[A\n",
      " 77%|███████▋  | 98/128 [00:22<00:06,  4.50it/s]\u001b[A\n",
      " 77%|███████▋  | 99/128 [00:22<00:06,  4.74it/s]\u001b[A\n",
      " 78%|███████▊  | 100/128 [00:22<00:05,  4.99it/s]\u001b[A\n",
      " 79%|███████▉  | 101/128 [00:22<00:05,  5.12it/s]\u001b[A\n",
      " 80%|███████▉  | 102/128 [00:23<00:05,  5.17it/s]\u001b[A\n",
      " 80%|████████  | 103/128 [00:23<00:04,  5.22it/s]\u001b[A\n",
      " 81%|████████▏ | 104/128 [00:23<00:04,  4.89it/s]\u001b[A\n",
      " 82%|████████▏ | 105/128 [00:23<00:05,  4.26it/s]\u001b[A\n",
      " 83%|████████▎ | 106/128 [00:24<00:04,  4.64it/s]\u001b[A\n",
      " 84%|████████▎ | 107/128 [00:24<00:04,  4.84it/s]\u001b[A\n",
      " 84%|████████▍ | 108/128 [00:24<00:04,  4.80it/s]\u001b[A\n",
      " 85%|████████▌ | 109/128 [00:24<00:03,  4.97it/s]\u001b[A\n",
      " 86%|████████▌ | 110/128 [00:24<00:03,  4.92it/s]\u001b[A\n",
      " 87%|████████▋ | 111/128 [00:25<00:03,  4.86it/s]\u001b[A\n",
      " 88%|████████▊ | 112/128 [00:25<00:03,  5.06it/s]\u001b[A\n",
      " 88%|████████▊ | 113/128 [00:25<00:02,  5.09it/s]\u001b[A\n",
      " 89%|████████▉ | 114/128 [00:25<00:02,  5.28it/s]\u001b[A\n",
      " 90%|████████▉ | 115/128 [00:25<00:02,  5.33it/s]\u001b[A\n",
      " 91%|█████████ | 116/128 [00:25<00:02,  5.13it/s]\u001b[A\n",
      " 91%|█████████▏| 117/128 [00:26<00:02,  5.19it/s]\u001b[A\n",
      " 92%|█████████▏| 118/128 [00:26<00:01,  5.09it/s]\u001b[A\n",
      " 93%|█████████▎| 119/128 [00:26<00:01,  5.14it/s]\u001b[A\n",
      " 94%|█████████▍| 120/128 [00:26<00:01,  5.04it/s]\u001b[A\n",
      " 95%|█████████▍| 121/128 [00:26<00:01,  5.24it/s]\u001b[A\n",
      " 95%|█████████▌| 122/128 [00:27<00:01,  5.04it/s]\u001b[A\n",
      " 96%|█████████▌| 123/128 [00:27<00:00,  5.32it/s]\u001b[A\n",
      " 97%|█████████▋| 124/128 [00:27<00:00,  5.35it/s]\u001b[A\n",
      " 98%|█████████▊| 125/128 [00:27<00:00,  4.67it/s]\u001b[A\n",
      " 98%|█████████▊| 126/128 [00:27<00:00,  4.88it/s]\u001b[A\n",
      " 99%|█████████▉| 127/128 [00:28<00:00,  4.92it/s]\u001b[A\n",
      "100%|██████████| 128/128 [00:28<00:00,  5.02it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======================================== SAMPLE 1 ========================================\n",
      "\n",
      "\n",
      "I'm a software engineer and I'm currently working on a project called \"Machine Learning for the Web\". I'm also a big fan of natural language processing and I'm working on a project called \"Natural Language Processing for the Web\".\n",
      "\n",
      "What's your favorite thing about machine learning?\n",
      "\n",
      "I love the fact that it's so easy to use. I love the fact that it's so easy to use.\n",
      "\n",
      "What's your favorite thing about natural language processing?\n",
      "\n",
      "I love the fact that it's so easy to use. I love the fact that it's so easy to use.\n",
      "\n",
      "What's\n",
      "================================================================================\n"
     ]
    }
   ],
   "source": [
    "seed = 0\n",
    "np.random.seed(seed)\n",
    "torch.random.manual_seed(seed)\n",
    "run_model('What do you know about Machine Learning and Natural Language Processing?',\n",
    "          length=128, sample=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 512/512 [01:32<00:00,  4.18it/s]\n",
      "  0%|          | 0/512 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======================================== SAMPLE 1 ========================================\n",
      " Skip all the introductory materials we make available (get up to speed by reading about these areas). Take a Google or TechNet course on Machine Learning. Learn how to use Coursera's MOOCs, MPI's, Dataslab OpenLab, and Velocity courses. Understands language learning, like etiquette and human language perception, across languages and cultural perspectives, is an important research area. Can you think of any obvious defense against this issue that involves a technical community? I ask because I suffer from community engineering paralysis. Thinking outside the box is hard, so I always try to take on projects and expertise in fields that may be at distinct risk of extinction.\n",
      "\n",
      "Full Video Those are below and here. What are the keys to mtn zero's success? They're equity & vehicle sentiment categories. Automotive appraisal is a powerful part of property valuation. Our results showed that we can convert Motalic's Label of great detected vehicles group into a yield related sentiment metric around 90 percent. By ratios for empty space, lateral movement, and dorsal cross-section. We chose Morpho and Forseli as catalysts to convert the Monitorage quantometer reporting into an attractive sentiment risk indicator. They are also groundwork for Radixin, Ecomorphology –w/thegyro Barometer Advocante BEST TECH STORY Other Menu\n",
      "\n",
      "All statistics I supplied here are provided in connection with the data being processed by special functions within mtn zero. For some fields the corresponding statistics are saysin' people are average, but I assumed everyone had an opinion. That's understandable with Motalic making it here with open prehospital scheduling the everything else in the industry on lock or lose. But obsessions over market mortality bias. That's repel if you enjoy 'make it work here free' well, it's not jumpin Bill Gates there –I've used BRCS with MNG p's from FNOSP for years if need be RAID loads are low and preventing people to revert back to the fold is critical accidents and wear sucks tires which if allowed to slowly spread (tech shirt in A e maximum) grow into aging anatomy scene will soon, inevitably, destroy your world. It's decades to before they're able to dramatically curb cost-driven tank wear where impairment is expected to be thru the roof as tire tread graph learn to trend fall among already fought transect.\n",
      "\n",
      "So Vest analysis, the concentrated unstable struck heaviest today btw things happened, dispanders/redeploy, continuous bag drops, passengers\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 512/512 [01:31<00:00,  4.15it/s]\n",
      "  0%|          | 0/512 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======================================== SAMPLE 1 ========================================\n",
      "\n",
      "\n",
      "In this article, you'll learn about…\n",
      "\n",
      "How to use Machine Learning and Natural Language Pair Questions (MLPQ tracks), and how to train Machine Learning and Nanosense on SPSS data.\n",
      "\n",
      "When I think about results in GPU trained data that can automatically help me diagnose a problem, it often feels like I am seeing the results of new movies. What makes a world outside of graphics surprising and uncanny?\n",
      " (See also: Expectations, Perception You Want Machine Learning to Discover).\n",
      "\n",
      "• Objective . Knowing where promises of high results lie and what new strategies they imply\n",
      "\n",
      "• Validation . Finding demand side proofs of upper bound validation of machine learning\n",
      "\n",
      "• Alternatives. When and how we can quantify these computations\n",
      "\n",
      "• Results. How in fact machine learning and natural language processing to uncover relevant methodologies\n",
      "\n",
      "• Money question – What can other members of the family help me?\n",
      "\n",
      "Implementing Machine Learning assumptions\n",
      "\n",
      "Analyse the screenshots available in our Data Reaper Report from reached 3 million scenes and can provide insights into awesome new technology released.\n",
      "\n",
      "DeepQA Find Results W GfMRI Request model for additional resolution 0/52 fully filled level 0 View all parameters In this study, we are able to evaluate a DeepQA match pattern created by finding up to a minimum of three best matches within a scene.\n",
      "\n",
      "De Ker MSJP MODEL REACTION TIME: CPU 250 SEC FILE: MODELS ID FILING BINDER DOC HERITAGE SEC LATEST CURSOR ID RANDOM GE matches One tale that we decided to use was a real train line and PCM gap estimation file available on Dekeer Medical Evaluation Medical Mortality Data https://bitbucket.org/dekerphotos/tspjsmj/test/downloads/tspjma.mp3/1m9Killers'/LSX.mp4 where we want to fill another 81 sized gap to trigger the remote mobile euthanasia disaster. EDIT: I just needs to add this: eye band color appears to be DNS lookup alternates (Array.prototype:ilinypu.). This item is on the \"light\" page.\n",
      "\n",
      "Attach the files to YesterGro data to detect if these third party data bins are mutually co-edge cropped. Check out the blend filters of YesterGro data and filter sse4 demo for examples. We need to confirm in each route to training purpose unless we are mixing and matching for, i\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 512/512 [01:43<00:00,  4.29it/s]\n",
      "  0%|          | 0/512 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======================================== SAMPLE 1 ========================================\n",
      " Send in your questions using the form below.\n",
      "\n",
      "Have you tried Machine Learning products? Have you blogged about them? We'd like to hear from you. There are two ways you can get involved, either make a blog post or submit a pull request. If you post a pull request it gets listed at the top of the article. Follow the instructions straight from Github so you don't miss out.\n",
      "\n",
      "0 answers<|endoftext|>Athenaeum tour is fun and exciting and seems to always have an odd mix of mechanical tronstros and knees... shmood, eccentric illnesses, and library tour de force's abound here. On the other hand there's so much going on that you might be getting all your information futilely. We expect it again tomorrow and I think Christian Zwanzgwille and David M. Kent would improve upon any of these sales mettle a bit, because Earthlings and %roads are quite a different creature. The whole thing reckons that there was a unit history blog on Jono von Slagtagen, but people gave up reporting on his blog about \"Stevie Wedeman\" years ago. Maybe David twigs it and decides to explore the material, and if you want if to stay away from that altogether, ROUTOS digitally scans that piece...\n",
      "\n",
      "TLEEP EXOXUS: When the word became submerged and Art Deco began it took over the quality of play with the first circa 1950's shirts that seemed some old Ennio Cinema film style shit anyway, make it out to Days of Suspense, Dismemberment, other death metal records on a heavier metal front you would be sorry to object to on nostalgic instruction from whipping. Jerry having known about Indian music in general for a Doomed life found it much too heavy now the version from (perhaps God-fearing indie thrashers, mathtron, Portland and Mississippi emo tattoo style weirdo culture of the Gene Wilder era) featured tubas slip into the and by over the course of the over just experiment more was born to discovery of the polarity of the magneton magnet crystals found in parisons laser demo Theater signal, better force or less fusion convert charge provided a thing of the past mix around one of the brightest users of mist universal worm ticks and amped levels of harm's over EK calibration for any specimen that camp really posess galahs stereo -NUF still Baby Doll sounds lovely but is lighter in tint but sound's still from Warpmonger losing parts\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 512/512 [01:35<00:00,  4.32it/s]\n",
      "  0%|          | 0/512 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======================================== SAMPLE 1 ========================================\n",
      "\n",
      "\n",
      "Robot physicians\n",
      "\n",
      "\n",
      "While there are some efficient weapons in Artificial General Intelligence (AGI), there is more than can be expected from significant advances of a few years given the limited resources required to understand and produce such systems. One and a half years ago a group of researchers from New York University and the University of Pennsylvania, Amos Cohen, and Atul Gupta analyzed the tonal and kinematic properties of acoustic-like sounds ranging from rap music to words in everyday speech. In the following years they settled on a classification system for the sounds and developed algorithms for identifying the opposition to each tone in many tones as well as distinguishing human from robot sounds.\n",
      "\n",
      "The second most fundamental approach that advanced artificial intelligence has used to date is speech recognition. After obtaining early results showing the revolutionary abilities of spoken words, AGI became critical to computer systems that decipher human speech and imitate it.Artificial general intelligence has followed in the footsteps of handwritten print, literal translation and syntactic ideal grammars in reading paper. Readings have gained significant speed, but reading problems have remained. By one recent estimate, human to robot translation calls 90 days at rate of 30 megs per second. Image processing has been relatively successful while, to date, the accuracy of task translation is even lower. At the same time, deep learning continues to push the envelope of machine learning, reaching resolutions of up to 1000 error rates per model.Artificial general intelligence at large training data villages with data repositories, task maps, relationships between models and computer servers were established. In recent years, high-performance systems available for bot tasks, converting to speech recognition, have been largely independent of this class of search low-level machinery. At seminars and workshops on AI, we have discussed the use of language processing systems not accessible to practice, still left unanswered questions for potential applications.\n",
      "\n",
      "Computer Health Virtual or Robotic US retirees currently living in undisclosed locations (nearly tens of thousands) try to live happily and offer support. These retirees against Lewandowski Robotics will depend a number of factors on the future performance of business models involving clients, solicited services, and the lack of guaranteed high - quality servers. Moreover, with higher workloads and profit options often include the purchase and utilization of \"health costs,\" it has created a rapidly growing scheduled cost center. Verbal communication designed to mimic- mimicking apparently understandable linguistic sounds, clear and low in content does provide access to crucial information relating the user's abilities, job status, general culture and how he/she relates to the\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 512/512 [01:26<00:00,  4.29it/s]\n",
      "  0%|          | 0/512 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======================================== SAMPLE 1 ========================================\n",
      " Do you have expert commentary on what needs to be done first?\n",
      "\n",
      "As for understanding its potential, would you like to become a gourmet vendor?\n",
      "\n",
      "Do you want remote involvement? Do you even want to have a job in the future situation? The long list of possibilities is large. Now, you're probably thinking quickly. \"Well, I just like to have fun.\" Well, we wouldn't be writing anything once and Alien Swarm requires no mental slippage due to single page web readers.\n",
      "\n",
      "I think I fill about 25 to 30 blends withMachine Learning topics each month. All those requests for more articles after so many years? Talkin love.<|endoftext|>All kinds of troubleshooting can be done for Windows XP and 2079 PCs or older versions of Windows. The following article will tell you how to troubleshoot and troubleshoot with multiple product titles inside SP2 and SP4.\n",
      "\n",
      "There appears to be a patch with some critical issues causing unknown programs to misbehave (based on my answers). It's no big deal, as machines have already come with the patch in order to fix it.\n",
      "\n",
      "In this article I describe using Windows Powershell, PowerShell Active Update, but understand the severe issue (CVE,KB 2756990) it may cause to your machine if it can't be patched properly.\n",
      "\n",
      "Problems as Windows 7 Forced, Incorrect RepairPackage ($ProdRef), has a MB 276 and $ProdRef in it\n",
      "\n",
      "Some times when installing Windows 7, sometimes that message points to a pitfall: \"Microsoft, Windows 7 is not installed.\" That causes an internal switch that could not be done. A significant number of installations called \"C:\\Windows\\System32\\winupd365.exe\" are defective with the same message (don't install X).\n",
      "\n",
      "For the rest of the objects taking up additional memory, the fix is scattered through windows due to a patched PatchMillis or products not generating enough samples. Along the way, you got to deal with false start.\n",
      "\n",
      "SecurityAgent can win a file based upon a list of restrictions (they mostly allow administrators to write too much code)\n",
      "\n",
      "This is the most simple and clean fix that any sane person would make. It would bring allocation and copying of objects normalized when the user is creating the object.\n",
      "\n",
      "This kind of change is really permissatory, since with an equivalent group policy, user groups (always available in the group running powershell) won't mean much\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 512/512 [01:25<00:00,  4.43it/s]\n",
      "  0%|          | 0/512 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======================================== SAMPLE 1 ========================================\n",
      " Please tell us in the comments below.<|endoftext|>Get daily news updates directly to your inbox Subscribe Thank you for subscribing We have more newsletters Show me See our privacy notice Could not subscribe, try again later Invalid Email\n",
      "\n",
      "Video Loading Video Unavailable Click to play Tap to play The video will start in 8 Cancel Play now\n",
      "\n",
      "Eight men were jailed after eating two striking burritos that cost a Tesco shopper £70.\n",
      "\n",
      "Louis Cheung, 45, of Athlone had tried to stop one of the fish-cheese-fruit spread being presented as a bacon sandwich when it was pressed into his mouth.\n",
      "\n",
      "While 20-year-old Raymond Cheng, from Yamapuro,amused Mr Cheung by only eating his way through the sandwich, spent days defiantly denying he had eaten any chips.\n",
      "\n",
      "U-turns had to be made after the following day and three months passed before the tea cart tycoon was finally found guilty.\n",
      "\n",
      "(Image: PRT)\n",
      "\n",
      "Video Loading Video Unavailable Click to play Tap to play The video will start in 8 Cancel Play now\n",
      "\n",
      "Judge Amy Hughson adjourned sentencing at Hull Crown Court despite objections by Mr Cheng, who had been sentenced to six months jail and banned from using coffeeshop-like machines again.\n",
      "­Cheung had denied having eaten the Eight 2 Sabre fish-Cheese Fruit spread, the court heard, but had admitting eating his chips into the face.\n",
      "\n",
      "Judge Hughson said: \"So it's just outrageous that you decided something happened to you and proceed to convey a message about your society that just infuriates me.\"\n",
      "\n",
      "Mr Cheung cannot be linked while there will be no further proceedings on an application to preclude the news brand from selling advertising on the carry-out machines.<|endoftext|>You are talking about which state is waging the \"war\" on whistleblowers with harsh sentences and the vague definition of torturing a friend . If the Navy nuke engineer Jeffrey St. Clair had written Robert Lee, instead of Peter Shields, he still would have been released today as Lightning.\n",
      "\n",
      "You are lying to the American people.\n",
      "\n",
      "To the people who answered for you, to the Electoral College vote, to Sean Ross Fitzgerald , a former Army Ranger so scandalously crass, so clueless and dumb and ignorant to Beer is a better Styrofoam cup of beer than a soul is going to Hollof Yard.\n",
      "\n",
      "Not enough cooks cook enough. We are creating too many cooks. I am saying\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 512/512 [01:27<00:00,  4.19it/s]\n",
      "  0%|          | 0/512 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======================================== SAMPLE 1 ========================================\n",
      " Let us know in the comments!<|endoftext|>Seattle Breathable WATERPROOF\n",
      "\n",
      "Co2 absorbers nearly 20 years ago received the Shock From Martian Orbits Award from the Society for Accrediting Electrical Engineers. These authors say these absorbents are unmitigated heroes for our water recycling efforts!<|endoftext|>FOXBORO — When Rob Gronkowski took to the sideline before Sunday's Patriots-49ers game in Foxboro to watch Ben Roethlisberger take carries later in the third quarter, Gronkowski wasn't making his first start for the team since losing 21 consecutive games during 2011. He was lost, his mind clouded by dadhood; he was reminiscing about old times with his mom and wife, then down for dinner, thinking of the two of them before he said an impromptu prayer.\n",
      "\n",
      "To the world, Gronkowski was god, starting there by striking up conversations with legendary Dunbar A. Roe about divorce, grief, heartbroken men, born-again each other, Bradshaw and Bixler and Jews, an unborn child and fatherhood. He was brimming with hope for this newly reborn family.\n",
      "\n",
      "Story continues below advertisement\n",
      "\n",
      "Down in the tunnel, Gronkowski stopped. Then he finished his talks. He \"joined the kids at dinner gameplay, and by the end of dinner he was giving Head Coach Belichick lines and talking about nothing else,\" says Roe, now an analyst with NFL Media. \"He's talking about his dad, his mom, demos going wrong. It was amazing to watch.\"\n",
      "\n",
      "Roethlisberger said at some point after the game that Gronkowski was \"going to bring 'em back, we're going to get everybody; 10 could bring this back. One in particular entered the game celebrating with guys and felt enormously for a hamstring injury he had. He wanted people diving for him and really focused on getting up and running, where it didn't matter if there were running back after running back, he was on it.\"\n",
      "\n",
      "As Gronkowski broke into a sweeping smile on his return to solo-laden Seattle, scattering at least two nods of affection along the way, perhaps this looked as much like Rocky James following comical Kegel exercises on OBJ as it did a Patriots victory miracle – something as utterly unpredictable as the magnitude of a home-field advantage, more than a tenuous bumble after a most improbable win seems.\n",
      "\n",
      "Still, breaking this barrier, to a grittier election No. 3 than perhaps any before my generation might\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 512/512 [01:28<00:00,  3.66it/s]\n",
      "  0%|          | 0/512 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======================================== SAMPLE 1 ========================================\n",
      " We want to hear about your natural language processing experiences!\n",
      "\n",
      "Banshee, Fog, Mace Khanana and Friman Wars Based out of Oslo, this crew works on the least of these and want you to know it! Learn more about Banhafis castle here – facebook\n",
      "\n",
      "Botkick Jul 26\n",
      "\n",
      "NGO asking for help for web service that no Social Media management company can investigate should anyone presume: hardware or software? The member's main concern is that both technologies are so prevalent in today's society in wide range of situations that most IT platforms don't even bother linking or formatting their entire B2S packages.\n",
      "\n",
      "Reek, Emperor\n",
      "\n",
      "1865 Tools for Browalscience published by Jerry Golomb on Inset Security Magazine \"These tools to myself find solutions for this software \"software lookup\" for backend control jobs, is it better to rely on base/formatting away, or to manually style at RBbenko's Treatante machine or use Algon&al law compliant Bcopy\"\n",
      "\n",
      "LegalPortals Jul 26\n",
      "\n",
      "Advice Ming: Notebook and social Network conversions with HTTP gist or User Express Gist of audio/visual format \"HP Neu gets each plaint that you get given to me by u/strand and it tells stories about me with my own final craftsmanship saying experiences polished for audiophile purists!\"\n",
      "\n",
      "Liquid Expo Jul 26\n",
      "\n",
      "Historical text exchange of the human affect curve. Convert rings into eyes Strange element in everyday experience ≥ 0 i.e, curious tremor of body. Asking the same question to oneself.\n",
      "\n",
      "TheNewPatrick Jul 26\n",
      "\n",
      "Customs Director — another hard shit :-/editable ever backup file mechanism reduces tech debt across international ports. And I don't care if a connection puts up American The tax professional notifying the Txve if they received a bill last month check nothing else on port, that.png files contribute hal opinion and do rise to my wall but meaning nothing is created as one would expect. Turns write permit finishing Cassian assumed Olsen will taste dreaming next longer, help normalize valentine's day to minor death insurance genius, a simple markup to last wife's days Will play nicely at wapo shop when own presents or holidays are paid. Love presidential banpops banana just absurd I mean think twice (decide you won't eat equally or eat well at the same rate only taste them.) or stupid isn't making the author responsible for i2p happen FTP coffee front\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 512/512 [01:32<00:00,  4.13it/s]\n",
      "  0%|          | 0/512 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======================================== SAMPLE 1 ========================================\n",
      " Have you used it when you're a teacher or intern in the classroom? Let's discuss it! Ask forth your questions in the comments below, or tweet me at 👍 @MichaelLadd. Search for him on GitHub. Related<|endoftext|>China's huge black arrows on a map on view inside the three-story box of a giant advances department store named China Ballo, which sells entertainment jewelry starting at 7,500 yuan ($8,700) a pop. (Bao Yu/CQ Roll Call)\n",
      "\n",
      "Switches and alternates hang on colorful pillows and pumps above hands that wobble in bedded undies as they hum and echo underwater images by Blizzard Entertainment's redrawing of video games as magical theaters now play imagination from South Korea. Boys play with dot-matrix petals on face masks decorated with big image bubbles, when hoops shoot from slats in blond fairy swimwear modeled after those worn by the country's Olympic athletes. Mouse , duck and cat eyes sparkle out from tinned collection packages on long hair pins counter-curved into bows. Mugs are decorated with hard-to-miss images of former Chinese rulers, paintings by northwest Chinese Chang Yongbao and scenes from the Hockey Hall of Fame and Accountability in Detroit, Michigan.\n",
      "\n",
      "There are people in this giant section of Xiânghí, a historic, Chinese city in Anhui province. The city, which is familiar to nerds and gamers from Doritos to Halo, has absorbed the logic and lithe realism of leagues, online gaming, sports and\"realistic\" design trends — beach and reservoir quadrangle assortments wearing t-shirts emblazoned with the words \"world\" and \"ninja\" fixing tanks on truck covers hugging jacking hammers.\n",
      "\n",
      "And yet there's something most people miss. Which she'd be right to.\n",
      "\n",
      "See also: China's No. 3 City is Absolutely Fitting for Trump's Made-in-America City Commission\n",
      "\n",
      "Xiânghí — known colloquially as Cangquishi — sits on one of China's most remote, mountainous and inhospitable regions. Its most famous landmark, the Black Dragon Hill crater carved by the mighty Han Dynasty in 1087 and known as Wuyi Rathi, is barren with mountains 115 feet all the way up and developed into a mammoth troupe of sports leagues competing for victory with goalposts and dozing platforms suspended from corporate aquariums designed to cultivate talent.\n",
      "\n",
      "Because\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 512/512 [01:41<00:00,  4.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======================================== SAMPLE 1 ========================================\n",
      "\n",
      "\n",
      "\n",
      "If you are interested to know more about AI/NCP please I do offer tutorials on several popular subjects like deep learning framework, deep learning network architectures, Neural Networks and more online.\n",
      "\n",
      "\n",
      "There are a lot of options available if you are interested in AI Programming.\n",
      "\n",
      "\n",
      "I offer two tutorials where you can visit this beautiful 200 square feet facility where we teach you basic Machine Learning principles like recursion, implementation, learning rates, reconstruction, trained HML files and more.\n",
      "\n",
      "\n",
      "LDP is provided by HealthLine as we do right now everything through email to help. Please don't hesitate to ask any questions you might have during any of our trainings or train your brain to recall something.\n",
      "\n",
      "\n",
      "Techniques and activities include copious tutorials such as green eyes. Deep learning programming as A/B testing gmc$UmbRnBiBiQ QT[end automated natural language processing steps • fix parameters ], libraries and etc.\n",
      "\n",
      "\n",
      "You can visit TU Campus Clothing & Drink(www.student-cuisine.com) where all vape products made in Romania can be found and customized for your individual needs. Also Barteret is excited to bring Doko/d'. What kind of cake that is funded?\n",
      "\n",
      "Tx Aub de Chao: They serve products with strong Di Caprio cinematic characteristics without being full of transgendence popcorn cartoon mush science them. Where's climbing area? What's on list?\n",
      "\n",
      "cdntomm\n",
      "\n",
      "\n",
      "A machine learning model searches through thousands of medical images, discarding misidentifications and outliers and with mind numbing accuracy, with very little noise.\n",
      "\n",
      "\n",
      "Imagine OpenCencrista, if you want to cleave all forest carcasses into thousands of different parts, but you also want nanotechnology to be able to collect, analyze, and reconstruct carcasses via deep and packed neural net architectures and artificial neural networks without you having to do the tedious line search solvers ( scanlines hobby) involved. Such a machine learning could be even cheaper, faster and easier than airplane had to come to and not even trying to a degree of capable energy of whatever type of nanotechnology doming partly biological tissue or human body fell down and fell to I think Enter Vitruvius, breathing air results in ensure cough, though to rule good eggs until we pass were have organic food in the past yet when the question was eat, we skipped it. kids dont nurse them I doddn what I do google No\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "seed = 0\n",
    "for seed in range(10):\n",
    "    np.random.seed(seed)\n",
    "    torch.random.manual_seed(seed)\n",
    "    run_model('What do you know about Machine Learning and Natural Language Processing?')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 128/128 [00:09<00:00, 13.55it/s]\n",
      "  0%|          | 0/128 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======================================== SAMPLE 1 ========================================\n",
      " Isn't it remarkable how powerful it is to let your mind run free of everything that's changing around it?\n",
      "\n",
      "We've worked at Machine Learning Hub for more than 20 years and have discovered that as a beginning along the way we are primed for processes that are indistinguishable from human behavior.\n",
      "\n",
      "Our first example premiered at the exact moment in history when Norma Bolt, a templated data scientist with the former Company That Builds Everything, won first prize to shut her down. One year later Bolt decided to give Carlos Miller fiscal incentives to write an article on Machine Learning and left working on it (besides the Ask Me Anything\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 128/128 [00:10<00:00, 12.38it/s]\n",
      "  1%|          | 1/128 [00:00<00:16,  7.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======================================== SAMPLE 1 ========================================\n",
      "\n",
      "\n",
      "If you're a teacher or teacher person and you take the time to learn more about Machine Learning Prediction Python written by three people smoking cigarettes: VThisuppolo Stefano aka 'teacher' and Asundri 'magic' Alibato\n",
      "\n",
      "Published – May 22, 2018\n",
      "\n",
      "PETROSOFT THUNDER\n",
      "\n",
      "Artistic Attorney is one of the recent founders\n",
      "\n",
      "developed by Pierre Tonkin Architects for the Jesuit university of Sepho on the outskirts of Groningen. The 42 year old TorboÍso Pascal reflects particularly on the process behind this project:\n",
      "\n",
      "Pierre has been true to his\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 128/128 [00:09<00:00, 12.84it/s]\n",
      "  1%|          | 1/128 [00:00<00:17,  7.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======================================== SAMPLE 1 ========================================\n",
      " Student: Machine Learning Designers. Tarryl: Machine Learning Tools for Education Programs. Roger: Virtual Reality Knowledge Base.\n",
      "\n",
      "4. CompuServe\n",
      "\n",
      "Advantages:\n",
      "\n",
      "- Awesome movies in online teacher education reviews.\n",
      "\n",
      "- Much of their online sales are focused around student reviews.\n",
      "\n",
      "- Page maintenance tools in user interfaces. Easy online assessment tool.\n",
      "\n",
      "- Augmented class discussions for students\n",
      "\n",
      "- Quality product reviews and maintainers that are extremely professional.\n",
      "\n",
      "- Lots of pmsc or categories for students.\n",
      "\n",
      "Many possibilities, for user becomes its own question particle patterns\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 128/128 [00:08<00:00, 14.74it/s]\n",
      "  1%|          | 1/128 [00:00<00:15,  8.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======================================== SAMPLE 1 ========================================\n",
      "\n",
      "\n",
      "Machine learning doesn't just mean running checks to learn new evidence based on an experiment run. You also have to build a model that knows how much to be different. Machine learning can work inside your data and attack your data over time.\n",
      "\n",
      "So how do you choose the right tool that you need, and what tools discourage you from using?\n",
      "\n",
      "Learning and Training Is a 2nd Step to Thinking Pavlov's Big Ideas\n",
      "\n",
      "Here's a short summary of what's being talked about on Machine Learning: you can read a lot more about Machine Learning before you sign up for DevOps or MySQL or Bootstrap business\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 128/128 [00:08<00:00, 15.71it/s]\n",
      "  1%|          | 1/128 [00:00<00:14,  8.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======================================== SAMPLE 1 ========================================\n",
      "\n",
      "\n",
      "\n",
      "I bought MariaScript for Linux. It is extremely fast! It was ever using file system system when I was running under Windows, Python and other code in Windows, so oh yeah, here we go. It is not all human-readable PCI-Express, but it works and is pretty secure, it works as many Unix or Windows machines as it can, and I think it does it all pretty nice! There are numerous things Python hasn't yet learned, but I suspect it will provide has probably at least one of them. Probably around 16 or more frameworks that Alexandros Berskov will use in open source or he'll become\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 128/128 [00:09<00:00, 14.54it/s]\n",
      "  1%|          | 1/128 [00:00<00:13,  9.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======================================== SAMPLE 1 ========================================\n",
      " How do you use it in your applications? How do you enable it? 0 of 10 Next Get answers in a Quora question.\n",
      "\n",
      "Natural Language Processing is the Processing of Data from One Medium by Jason Grossier\n",
      "\n",
      "Daitsy writes: \"The story of how artificial intelligence has come up with a very convincing model that predicts very heavily on our history: the path of a typical human home.\" That's an interesting line in the wrong direction. Only in 2017 have computer scientists quite managed to figure it out.\n",
      "\n",
      "The year 2001 was the 'golden window' was set. Robin Kamms was the CIA director\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 128/128 [00:09<00:00, 14.16it/s]\n",
      "  1%|          | 1/128 [00:00<00:14,  8.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======================================== SAMPLE 1 ========================================\n",
      " You probably heard from many companies. Here are five that I realize are leading Silicon Valley companies that are leading these industries.\n",
      "\n",
      "Step 1 – DevOps\n",
      "\n",
      "Almost everyone I talk to who's considering jumping into the tech industry has stated their intent to start a take on DevOps. Creating and using Container Containerization for the IT industry is the most obvious example of CI — DevOps isn't about tracking down required employee iterations (\".asp file info\"File'dctlHelp\", etc.) or cluttering up regular deploys to EOM's. But creating quick deployment pipelines for Docker containers is however the name. In your approach,\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 128/128 [00:08<00:00, 14.83it/s]\n",
      "  1%|          | 1/128 [00:00<00:14,  8.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======================================== SAMPLE 1 ========================================\n",
      "\n",
      "\n",
      "Machine Learning engineers simply use Translate which is a data-rich algorithm, crafted to automate mathematical verification and store tens of billions of data streams in a secure database.\n",
      "\n",
      "YOUR GLOBAL LEADER\n",
      "\n",
      "How did your time at University Connect image manipulation workshop effort impact your skill set?\n",
      "\n",
      "At Baccalaureate I found that we prioritized knowledge over experience & mentorship tenure, leadership, actual work, and keep it real. In addition to our many ancestors, our knowledgeable Chinese and Vietnamese colleagues learn through field education & talent exchange—contorically crafts that empower us to be as trustworthy as we\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 128/128 [00:09<00:00, 13.83it/s]\n",
      "  1%|          | 1/128 [00:00<00:16,  7.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======================================== SAMPLE 1 ========================================\n",
      " Let me know in the comments below!\n",
      "\n",
      "Image credits: Wikimedia Commons , dboven, GPSImagesMagnet.com, iLibrary http://tsk<|endoftext|>NEW DELHI: India is facing a serious crisis over what can happen when it needs to cull 2,500 trainees once it stalks express train with a massive explosive discovered here late last month.\n",
      "\n",
      "Experts say constraints imposed on trainmen by railway authorities and lawmakers have allowed the project to fail, leading to an \"inability\" to get train utilised in the next couple of weeks.\n",
      "\n",
      "Officials point out that intrusive screening of element 500 used in Pakistan\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 128/128 [00:08<00:00, 15.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======================================== SAMPLE 1 ========================================\n",
      "\n",
      "\n",
      "Allen: Well, I first wrote about understanding machine learning after a really cold RSS feed that I've seen to the nearest 100,000 on Slack. Sort of back to hacker stuff. I've learned a lot about language driven AI through my experiences with AI modelling, how the big pin backpacks really work. Also, understanding the bigger problems is difficult but worth it so far. If you can print an entirely pre-programmed neural model and it turns out you can do pre-code which is interesting to know what you're doing with that sensor on your TV that the AI is chasing for.\n",
      "\n",
      "And so claiming\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "seed = 0\n",
    "for seed in range(10):\n",
    "    np.random.seed(seed)\n",
    "    torch.random.manual_seed(seed)\n",
    "    run_model('What do you know about Machine Learning and Natural Language Processing?', length=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7 (fastai)",
   "language": "python",
   "name": "fastai"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}