test/casrel_bert.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4c72901",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import os\n",
    "import json\n",
    "import jieba\n",
    "import torch\n",
    "import pickle\n",
    "import codecs\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import pandas as pd\n",
    "\n",
    "from ark_nlp.model.re.casrel_bert import CasRelBert\n",
    "from ark_nlp.model.re.casrel_bert import CasRelBertConfig\n",
    "from ark_nlp.model.re.casrel_bert import Dataset\n",
    "from ark_nlp.model.re.casrel_bert import Task\n",
    "from ark_nlp.model.re.casrel_bert import get_default_model_optimizer\n",
    "from ark_nlp.model.re.casrel_bert import Tokenizer\n",
    "from ark_nlp.factory.loss_function import CasRelLoss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69854942",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 目录地址\n",
    "\n",
    "train_data_path = '../data/source_datasets/CMeIE/CMeIE_train.json'\n",
    "dev_data_path = '../data/source_datasets/CMeIE/CMeIE_dev.json'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc598e6a",
   "metadata": {},
   "source": [
    "#### 1. 数据读入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77edd06b",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_list = []\n",
    "\n",
    "with codecs.open(train_data_path, mode='r', encoding='utf8') as f:\n",
    "    lines = f.readlines()\n",
    "    for index_, line_ in enumerate(lines):\n",
    "        record_ = {}\n",
    "        line_ = json.loads(line_.strip())\n",
    "        record_['text'] = line_['text']\n",
    "        record_['label'] = []\n",
    "        for triple_ in line_['spo_list']:\n",
    "            record_['label'].append([\n",
    "                triple_['subject'],\n",
    "                record_['text'].index(triple_['subject']),\n",
    "                record_['text'].index(triple_['subject'])+ len(triple_['subject']) - 1,\n",
    "                triple_['predicate'] + '@' + triple_['object_type']['@value'],\n",
    "                triple_['object']['@value'],\n",
    "                record_['text'].index(triple_['object']['@value']),\n",
    "                record_['text'].index(triple_['object']['@value']) + len(triple_['object']['@value']) - 1,\n",
    "            ])\n",
    "        train_data_list.append(record_)\n",
    "\n",
    "train_df = pd.DataFrame(train_data_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a443cad",
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_data_list = []\n",
    "counter = 0\n",
    "with codecs.open(dev_data_path, mode='r', encoding='utf8') as f:\n",
    "    lines = f.readlines()\n",
    "    for index_, line_ in enumerate(lines):\n",
    "        record_ = {}\n",
    "        line_ = json.loads(line_.strip())\n",
    "        record_['text'] = line_['text']\n",
    "        record_['label'] = []\n",
    "        for triple_ in line_['spo_list']:\n",
    "            record_['label'].append([\n",
    "                triple_['subject'],\n",
    "                record_['text'].index(triple_['subject']),\n",
    "                record_['text'].index(triple_['subject'])+ len(triple_['subject']) - 1,\n",
    "                triple_['predicate'] + '@' + triple_['object_type']['@value'],\n",
    "                triple_['object']['@value'],\n",
    "                record_['text'].index(triple_['object']['@value']),\n",
    "                record_['text'].index(triple_['object']['@value']) + len(triple_['object']['@value']) - 1,\n",
    "            ])\n",
    "            counter += 1\n",
    "        dev_data_list.append(record_)\n",
    "        \n",
    "dev_df = pd.DataFrame(dev_data_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24d51c3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "re_train_dataset = Dataset(train_df)\n",
    "re_dev_dataset = Dataset(dev_df,\n",
    "                         categories = re_train_dataset.categories,\n",
    "                         is_train=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eec01479",
   "metadata": {},
   "source": [
    "#### 2. 词典创建和生成分词器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41bfb854",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(vocab='nghuyong/ernie-1.0', max_seq_len=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67ecaa82",
   "metadata": {},
   "source": [
    "#### 3. ID化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29d01f62",
   "metadata": {},
   "outputs": [],
   "source": [
    "re_train_dataset.convert_to_ids(tokenizer)\n",
    "re_dev_dataset.convert_to_ids(tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8918cb18",
   "metadata": {},
   "source": [
    "<br>\n",
    "\n",
    "### 二、模型构建"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71ef3699",
   "metadata": {},
   "source": [
    "#### 1. 模型参数设置"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30173f8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_config = CasRelBertConfig.from_pretrained('nghuyong/ernie-1.0',\n",
    "                                               num_labels=len(re_train_dataset.cat2id))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af4442d6",
   "metadata": {},
   "source": [
    "#### 2. 模型创建"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8882050a",
   "metadata": {},
   "outputs": [],
   "source": [
    "dl_module = CasRelBert.from_pretrained('nghuyong/ernie-1.0',\n",
    "                                       config=bert_config)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5187f461",
   "metadata": {},
   "source": [
    "<br>\n",
    "\n",
    "### 三、任务构建"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "589f7157",
   "metadata": {},
   "source": [
    "#### 1. 任务参数和必要部件设定"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e6c7add",
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = get_default_model_optimizer(dl_module) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aeb86665",
   "metadata": {},
   "source": [
    "#### 2. 任务创建"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e822c7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Task(dl_module, optimizer, CasRelLoss(), cuda_device=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5931e90",
   "metadata": {},
   "source": [
    "#### 3. 训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47ab7d32",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.fit(\n",
    "    re_train_dataset,\n",
    "    re_dev_dataset,\n",
    "    epochs=40,\n",
    "    batch_size=6\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a13ba861",
   "metadata": {},
   "source": [
    "<br>\n",
    "\n",
    "### 四、模型预测"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e901faec-cc1d-4167-8166-9cf3992f8d03",
   "metadata": {},
   "source": [
    "#### 1. 模型验证"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3aaa3474",
   "metadata": {},
   "outputs": [],
   "source": [
    "from ark_nlp.model.re.casrel_bert import Predictor\n",
    "\n",
    "casrel_re_predictor_instance = Predictor(model.module, tokenizer, re_train_dataset.cat2id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e5eb128",
   "metadata": {},
   "outputs": [],
   "source": [
    "text = '骨性关节炎@在其他关节（如踝关节和腕关节），骨性关节炎比较少见，并且一般有潜在的病因（如结晶性关节病、创伤）'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2f70656",
   "metadata": {},
   "outputs": [],
   "source": [
    "casrel_re_predictor_instance.predict_one_sample(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f626067f-1d72-46e7-ad2b-40538ec0b7ff",
   "metadata": {},
   "source": [
    "#### 2. 多样本验证"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca0954d5-17c1-43a4-83b1-cc67aec9efaf",
   "metadata": {},
   "outputs": [],
   "source": [
    "record_ = []\n",
    "with codecs.open('../data/source_datasets/CMeIE/CMeIE_test.json', mode='r', encoding='utf8') as f:\n",
    "    lines = f.readlines()\n",
    "    for index_, line_ in enumerate(lines):\n",
    "        line_ = json.loads(line_.strip())\n",
    "        record_.append([line_['text'], casrel_re_predictor_instance.predict_one_sample(line_['text'])])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb22c352-68c2-4c77-927d-4e9b988f5fbe",
   "metadata": {},
   "source": [
    "<br>\n",
    "\n",
    "### 五、模型测试报告"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4eb89fd-b7b4-4bb1-85f1-a2782c752d66",
   "metadata": {},
   "outputs": [],
   "source": [
    "1. 基本功能测试 通过\n",
    "2. one sample predict 通过\n",
    "3. 多样本验证  通过"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}