7.中文分类.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3362a434",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration lansinuote--ChnSentiCorp-4d058ef86e3db8d5\n",
      "Reusing dataset parquet (/root/.cache/huggingface/datasets/lansinuote___parquet/lansinuote--ChnSentiCorp-4d058ef86e3db8d5/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(9600,\n",
       " ('选择珠江花园的原因就是方便，有电动扶梯直接到达海边，周围餐馆、食廊、商场、超市、摊位一应俱全。酒店装修一般，但还算整洁。 泳池在大堂的屋顶，因此很小，不过女儿倒是喜欢。 包的早餐是西式的，还算丰富。 服务吗，一般',\n",
       "  1))"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "from datasets import load_dataset\n",
    "\n",
    "\n",
    "#定义数据集\n",
    "class Dataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, split):\n",
    "        self.dataset = load_dataset(path='lansinuote/ChnSentiCorp', split=split)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.dataset)\n",
    "\n",
    "    def __getitem__(self, i):\n",
    "        text = self.dataset[i]['text']\n",
    "        label = self.dataset[i]['label']\n",
    "\n",
    "        return text, label\n",
    "\n",
    "\n",
    "dataset = Dataset('train')\n",
    "\n",
    "len(dataset), dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e70a58c6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PreTrainedTokenizer(name_or_path='bert-base-chinese', vocab_size=21128, model_max_len=512, is_fast=False, padding_side='right', truncation_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'})"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import BertTokenizer\n",
    "\n",
    "#加载字典和分词工具\n",
    "token = BertTokenizer.from_pretrained('bert-base-chinese')\n",
    "\n",
    "token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e59695a4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "600\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(torch.Size([16, 500]),\n",
       " torch.Size([16, 500]),\n",
       " torch.Size([16, 500]),\n",
       " tensor([0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1]))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def collate_fn(data):\n",
    "    sents = [i[0] for i in data]\n",
    "    labels = [i[1] for i in data]\n",
    "\n",
    "    #编码\n",
    "    data = token.batch_encode_plus(batch_text_or_text_pairs=sents,\n",
    "                                   truncation=True,\n",
    "                                   padding='max_length',\n",
    "                                   max_length=500,\n",
    "                                   return_tensors='pt',\n",
    "                                   return_length=True)\n",
    "\n",
    "    #input_ids:编码之后的数字\n",
    "    #attention_mask:是补零的位置是0,其他位置是1\n",
    "    input_ids = data['input_ids']\n",
    "    attention_mask = data['attention_mask']\n",
    "    token_type_ids = data['token_type_ids']\n",
    "    labels = torch.LongTensor(labels)\n",
    "\n",
    "    #print(data['length'], data['length'].max())\n",
    "\n",
    "    return input_ids, attention_mask, token_type_ids, labels\n",
    "\n",
    "\n",
    "#数据加载器\n",
    "loader = torch.utils.data.DataLoader(dataset=dataset,\n",
    "                                     batch_size=16,\n",
    "                                     collate_fn=collate_fn,\n",
    "                                     shuffle=True,\n",
    "                                     drop_last=True)\n",
    "\n",
    "for i, (input_ids, attention_mask, token_type_ids,\n",
    "        labels) in enumerate(loader):\n",
    "    break\n",
    "\n",
    "print(len(loader))\n",
    "input_ids.shape, attention_mask.shape, token_type_ids.shape, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f620d0e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5f1a42f2a1c5435da295ad244cec4f4b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/393M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-chinese were not used when initializing BertModel: ['cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "torch.Size([16, 500, 768])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import BertModel\n",
    "\n",
    "#加载预训练模型\n",
    "pretrained = BertModel.from_pretrained('bert-base-chinese')\n",
    "\n",
    "#不训练,不需要计算梯度\n",
    "for param in pretrained.parameters():\n",
    "    param.requires_grad_(False)\n",
    "\n",
    "#模型试算\n",
    "out = pretrained(input_ids=input_ids,\n",
    "           attention_mask=attention_mask,\n",
    "           token_type_ids=token_type_ids)\n",
    "\n",
    "out.last_hidden_state.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5d3d02a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([16, 2])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#定义下游任务模型\n",
    "class Model(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.fc = torch.nn.Linear(768, 2)\n",
    "\n",
    "    def forward(self, input_ids, attention_mask, token_type_ids):\n",
    "        with torch.no_grad():\n",
    "            out = pretrained(input_ids=input_ids,\n",
    "                       attention_mask=attention_mask,\n",
    "                       token_type_ids=token_type_ids)\n",
    "\n",
    "        out = self.fc(out.last_hidden_state[:, 0])\n",
    "\n",
    "        out = out.softmax(dim=1)\n",
    "\n",
    "        return out\n",
    "\n",
    "\n",
    "model = Model()\n",
    "\n",
    "model(input_ids=input_ids,\n",
    "      attention_mask=attention_mask,\n",
    "      token_type_ids=token_type_ids).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1bd44a7c",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt36/lib/python3.6/site-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  FutureWarning,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0.7399377822875977 0.375\n",
      "5 0.6872416138648987 0.5\n",
      "10 0.6332055330276489 0.75\n",
      "15 0.6033581495285034 0.875\n",
      "20 0.5943211317062378 0.75\n",
      "25 0.6094914674758911 0.6875\n",
      "30 0.6384122371673584 0.6875\n",
      "35 0.5677111744880676 0.8125\n",
      "40 0.5644276142120361 0.875\n",
      "45 0.5818642377853394 0.8125\n",
      "50 0.5436866879463196 0.875\n",
      "55 0.5654083490371704 0.75\n",
      "60 0.6283971071243286 0.6875\n",
      "65 0.4929084777832031 0.875\n",
      "70 0.5428925156593323 0.8125\n",
      "75 0.5615941882133484 0.75\n",
      "80 0.49903589487075806 0.8125\n",
      "85 0.5711023211479187 0.6875\n",
      "90 0.5560241341590881 0.8125\n",
      "95 0.5039068460464478 0.75\n",
      "100 0.45021048188209534 0.9375\n",
      "105 0.5826895833015442 0.6875\n",
      "110 0.6238031983375549 0.625\n",
      "115 0.4193401634693146 1.0\n",
      "120 0.5034649968147278 0.875\n",
      "125 0.4235936105251312 0.9375\n",
      "130 0.42960819602012634 1.0\n",
      "135 0.37958282232284546 1.0\n",
      "140 0.5208486318588257 0.8125\n",
      "145 0.5100721716880798 0.8125\n",
      "150 0.43825772404670715 0.875\n",
      "155 0.4317510724067688 0.9375\n",
      "160 0.4287576675415039 0.9375\n",
      "165 0.4266912639141083 0.9375\n",
      "170 0.4405173361301422 0.875\n",
      "175 0.41791820526123047 0.875\n",
      "180 0.4508487582206726 0.8125\n",
      "185 0.4445679485797882 0.8125\n",
      "190 0.6184951066970825 0.625\n",
      "195 0.5331169962882996 0.75\n",
      "200 0.5569592714309692 0.75\n",
      "205 0.3625647723674774 1.0\n",
      "210 0.4005920886993408 0.9375\n",
      "215 0.4386642575263977 0.9375\n",
      "220 0.5098661184310913 0.8125\n",
      "225 0.447701096534729 0.9375\n",
      "230 0.3920193314552307 0.9375\n",
      "235 0.4688347280025482 0.8125\n",
      "240 0.46488767862319946 0.9375\n",
      "245 0.4752849340438843 0.875\n",
      "250 0.48376595973968506 0.8125\n",
      "255 0.4025830626487732 0.9375\n",
      "260 0.4019404351711273 0.9375\n",
      "265 0.5092278122901917 0.8125\n",
      "270 0.6099189519882202 0.625\n",
      "275 0.42547380924224854 0.875\n",
      "280 0.5527713894844055 0.75\n",
      "285 0.5336724519729614 0.75\n",
      "290 0.4221529960632324 0.8125\n",
      "295 0.38887107372283936 1.0\n",
      "300 0.4890342056751251 0.8125\n"
     ]
    }
   ],
   "source": [
    "from transformers import AdamW\n",
    "\n",
    "#训练\n",
    "optimizer = AdamW(model.parameters(), lr=5e-4)\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "\n",
    "model.train()\n",
    "for i, (input_ids, attention_mask, token_type_ids,\n",
    "        labels) in enumerate(loader):\n",
    "    out = model(input_ids=input_ids,\n",
    "                attention_mask=attention_mask,\n",
    "                token_type_ids=token_type_ids)\n",
    "\n",
    "    loss = criterion(out, labels)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    optimizer.zero_grad()\n",
    "\n",
    "    if i % 5 == 0:\n",
    "        out = out.argmax(dim=1)\n",
    "        accuracy = (out == labels).sum().item() / len(labels)\n",
    "\n",
    "        print(i, loss.item(), accuracy)\n",
    "\n",
    "    if i == 300:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "275dd1b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration lansinuote--ChnSentiCorp-4d058ef86e3db8d5\n",
      "Reusing dataset parquet (/root/.cache/huggingface/datasets/lansinuote___parquet/lansinuote--ChnSentiCorp-4d058ef86e3db8d5/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "0.85\n"
     ]
    }
   ],
   "source": [
    "#测试\n",
    "def test():\n",
    "    model.eval()\n",
    "    correct = 0\n",
    "    total = 0\n",
    "\n",
    "    loader_test = torch.utils.data.DataLoader(dataset=Dataset('validation'),\n",
    "                                              batch_size=32,\n",
    "                                              collate_fn=collate_fn,\n",
    "                                              shuffle=True,\n",
    "                                              drop_last=True)\n",
    "\n",
    "    for i, (input_ids, attention_mask, token_type_ids,\n",
    "            labels) in enumerate(loader_test):\n",
    "\n",
    "        if i == 5:\n",
    "            break\n",
    "\n",
    "        print(i)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            out = model(input_ids=input_ids,\n",
    "                        attention_mask=attention_mask,\n",
    "                        token_type_ids=token_type_ids)\n",
    "\n",
    "        out = out.argmax(dim=1)\n",
    "        correct += (out == labels).sum().item()\n",
    "        total += len(labels)\n",
    "\n",
    "    print(correct / total)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}