BAI2-BE/NER_model_trainer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Collecting en-core-web-lg==3.5.0\n",
      "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.5.0/en_core_web_lg-3.5.0-py3-none-any.whl (587.7 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m587.7/587.7 MB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:05\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: spacy<3.6.0,>=3.5.0 in /home/gabriel/.local/lib/python3.10/site-packages (from en-core-web-lg==3.5.0) (3.5.3)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /home/gabriel/.local/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (3.0.8)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /home/gabriel/.local/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (4.65.0)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /home/gabriel/.local/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (1.0.9)\n",
      "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /home/gabriel/.local/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (1.0.4)\n",
      "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /home/gabriel/.local/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (3.3.0)\n",
      "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /home/gabriel/.local/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (2.0.8)\n",
      "Requirement already satisfied: typer<0.8.0,>=0.3.0 in /home/gabriel/.local/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (0.7.0)\n",
      "Requirement already satisfied: jinja2 in /home/gabriel/.local/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (3.1.2)\n",
      "Requirement already satisfied: numpy>=1.15.0 in /home/gabriel/.local/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (1.24.3)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /home/gabriel/.local/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (2.0.7)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/lib/python3/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (2.25.1)\n",
      "Requirement already satisfied: pathy>=0.10.0 in /home/gabriel/.local/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (0.10.1)\n",
      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4 in /home/gabriel/.local/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (1.10.8)\n",
      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /home/gabriel/.local/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (3.0.12)\n",
      "Requirement already satisfied: thinc<8.2.0,>=8.1.8 in /home/gabriel/.local/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (8.1.10)\n",
      "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /home/gabriel/.local/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (6.3.0)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/gabriel/.local/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (23.1)\n",
      "Requirement already satisfied: setuptools in /usr/lib/python3/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (59.6.0)\n",
      "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /home/gabriel/.local/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (2.4.6)\n",
      "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /home/gabriel/.local/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (1.1.1)\n",
      "Requirement already satisfied: typing-extensions>=4.2.0 in /home/gabriel/.local/lib/python3.10/site-packages (from pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4->spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (4.6.3)\n",
      "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /home/gabriel/.local/lib/python3.10/site-packages (from thinc<8.2.0,>=8.1.8->spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (0.0.4)\n",
      "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /home/gabriel/.local/lib/python3.10/site-packages (from thinc<8.2.0,>=8.1.8->spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (0.7.9)\n",
      "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/lib/python3/dist-packages (from typer<0.8.0,>=0.3.0->spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (8.0.3)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /home/gabriel/.local/lib/python3.10/site-packages (from jinja2->spacy<3.6.0,>=3.5.0->en-core-web-lg==3.5.0) (2.1.2)\n",
      "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
      "You can now load the package via spacy.load('en_core_web_lg')\n"
     ]
    }
   ],
   "source": [
    "# Downloads the spacy model for the english language with a bash command\n",
    "\n",
    "!python3 -m spacy download en_core_web_lg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gabriel/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<spacy.lang.en.English at 0x7f456d931ed0>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Loads the model\n",
    "\n",
    "nlp = spacy.load(\"en_core_web_lg\")\n",
    "nlp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "# Load the training data\n",
    "\n",
    "with open('training data/ner spacy data/training_data_NER.json', 'r', encoding='utf-8') as f:\n",
    "    data = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'How many orders were placed in Lyon?'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Data example:\n",
    "\n",
    "data[15]['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'text': 'What is the total amount of payments received from Singapore?',\n",
       " 'entities': [(51, 59, 'CITY')]}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Creates a dictionary of training data for the NER model\n",
    "\n",
    "training_data = []\n",
    "for obj in data:\n",
    "    temp_dict = {}\n",
    "    temp_dict['text'] = obj['text']\n",
    "    temp_dict['entities'] = []\n",
    "    for ent in obj['entities']:\n",
    "        start = ent['startIndex']\n",
    "        end = ent['endIndex']\n",
    "        label = ent['tag'].upper()\n",
    "        temp_dict['entities'].append((start, end, label))\n",
    "    training_data.append(temp_dict)\n",
    "\n",
    "training_data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from spacy.tokens import DocBin\n",
    "from tqdm import tqdm\n",
    "\n",
    "# Load a new spacy model\n",
    "\n",
    "nlp = spacy.blank(\"en\")\n",
    "doc_bin_train = DocBin()\n",
    "doc_bin_val = DocBin()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4807/4807 [00:00<00:00, 13061.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of nulls: 0\n"
     ]
    }
   ],
   "source": [
    "from spacy.util import filter_spans\n",
    "import random\n",
    "\n",
    "\n",
    "# Saves the training data as a spacy binary file\n",
    "\n",
    "for training_example in tqdm(training_data): \n",
    "    text = training_example['text']\n",
    "    labels = training_example['entities']\n",
    "    doc = nlp.make_doc(text) \n",
    "    ents = []\n",
    "    x = 0\n",
    "    for start, end, label in labels:\n",
    "        span = doc.char_span(start, end, label=label, alignment_mode=\"contract\")\n",
    "        if span is None:\n",
    "            x += 1\n",
    "        else:\n",
    "            ents.append(span)\n",
    "    filtered_ents = filter_spans(ents)\n",
    "    doc.ents = filtered_ents\n",
    "    \n",
    "    rand = random.randint(1, 100)\n",
    "    if rand < 80: \n",
    "        doc_bin_train.add(doc)\n",
    "    else:\n",
    "        doc_bin_val.add(doc)\n",
    "\n",
    "doc_bin_train.to_disk(\"train.spacy\")\n",
    "doc_bin_val.to_disk(\"val.spacy\")\n",
    "print(\"number of nulls: \" + str(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;2m✔ Auto-filled config with all values\u001b[0m\n",
      "\u001b[38;5;2m✔ Saved config\u001b[0m\n",
      "config.cfg\n",
      "You can now add your data and train your pipeline:\n",
      "python -m spacy train config.cfg --paths.train ./train.spacy --paths.dev ./dev.spacy\n"
     ]
    }
   ],
   "source": [
    "# Creates a configuration file based on a preconfig downloaded from the official spacy website\n",
    "\n",
    "!python3 -m spacy init fill-config base_config.cfg config.cfg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;4mℹ Saving to output directory: spacy\u001b[0m\n",
      "\u001b[38;5;4mℹ Using CPU\u001b[0m\n",
      "\u001b[1m\n",
      "=========================== Initializing pipeline ===========================\u001b[0m\n",
      "[2023-07-08 02:13:12,818] [INFO] Set up nlp object from config\n",
      "[2023-07-08 02:13:12,831] [INFO] Pipeline: ['tok2vec', 'ner']\n",
      "[2023-07-08 02:13:12,834] [INFO] Created vocabulary\n",
      "[2023-07-08 02:13:17,503] [INFO] Added vectors: en_core_web_lg\n",
      "[2023-07-08 02:13:17,503] [INFO] Finished initializing nlp object\n",
      "[2023-07-08 02:13:19,383] [INFO] Initialized pipeline components: ['tok2vec', 'ner']\n",
      "\u001b[38;5;2m✔ Initialized pipeline\u001b[0m\n",
      "\u001b[1m\n",
      "============================= Training pipeline =============================\u001b[0m\n",
      "\u001b[38;5;4mℹ Pipeline: ['tok2vec', 'ner']\u001b[0m\n",
      "\u001b[38;5;4mℹ Initial learn rate: 0.001\u001b[0m\n",
      "E    #       LOSS TOK2VEC  LOSS NER  ENTS_F  ENTS_P  ENTS_R  SCORE \n",
      "---  ------  ------------  --------  ------  ------  ------  ------\n",
      "  0       0          0.00     55.82    0.00    0.00    0.00    0.00\n",
      "  0     200         17.27   1654.44   88.71   91.97   85.67    0.89\n",
      "  0     400         18.75    526.75   92.93   92.22   93.65    0.93\n",
      "  1     600         10.96    483.84   92.98   97.60   88.78    0.93\n",
      "  1     800         12.77    475.97   93.26   96.95   89.84    0.93\n",
      "  2    1000         14.96    579.86   94.15   91.42   97.04    0.94\n",
      "  3    1200         50.70    779.74   94.52   91.32   97.95    0.95\n",
      "  3    1400         16.80    777.55   93.33   98.21   88.92    0.93\n",
      "  5    1600         16.71    922.16   94.37   90.86   98.17    0.94\n",
      "  6    1800         15.15   1027.79   94.07   97.00   91.32    0.94\n",
      "  8    2000         22.97   1269.00   93.95   92.07   95.91    0.94\n",
      " 10    2200         18.15   1577.10   94.57   91.66   97.67    0.95\n",
      " 13    2400         17.65   1797.53   94.60   90.63   98.94    0.95\n",
      " 15    2600         29.35   1880.02   93.81   98.08   89.91    0.94\n",
      " 18    2800         19.34   1872.83   93.43   96.26   90.76    0.93\n",
      " 21    3000         16.00   1810.71   94.66   90.37   99.36    0.95\n",
      " 23    3200         23.51   1834.84   93.84   96.43   91.39    0.94\n",
      " 26    3400         20.06   1819.34   93.19   96.88   89.77    0.93\n",
      " 29    3600         19.46   1798.22   93.06   98.66   88.07    0.93\n",
      " 31    3800         19.32   1813.81   94.56   91.27   98.09    0.95\n",
      " 34    4000         39.90   1823.41   93.88   91.10   96.82    0.94\n",
      " 37    4200         29.61   1826.44   93.98   92.19   95.84    0.94\n",
      " 39    4400         26.29   1800.01   94.07   91.52   96.75    0.94\n",
      " 42    4600         43.35   1815.05   93.94   97.86   90.33    0.94\n",
      "\u001b[38;5;2m✔ Saved pipeline to output directory\u001b[0m\n",
      "spacy/model-last\n"
     ]
    }
   ],
   "source": [
    "# Trains the model using the training and validation data\n",
    "\n",
    "!python3 -m spacy train config.cfg --output ./spacy --paths.train ./train.spacy --paths.dev ./val.spacy"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}