1-2_bert_train.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "# BERT train\n",
    "\n",
    "작성일자: 210116 \\\n",
    "작성자: 조진욱\\\n",
    "목표: HF 기반 BERT 를 작은 SQuAD 데이터셋으로 Fine tuning 시켜보자\\\n",
    "비고: \n",
    "1. 학습속도를 위해 apex 를 사용함\n",
    "2. transfomers 패키지 내 squad_evaluate 가 실행시간이 너무 오래걸림. 왜지? 아직 모르겠음\n",
    "- 따로 테스트 해봤을땐 그렇게 느리지 않았던 걸로 기억\n",
    "\n",
    "레퍼런스 코드\n",
    "https://github.com/huggingface/transformers/blob/master/examples/legacy/question-answering/run_squad.py\n",
    "\n",
    "가급적이면 수정사항을 적어두려고함(TO DO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
    "from tqdm.autonotebook import tqdm, trange\n",
    "import os\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "import transformers\n",
    "from transformers import (\n",
    "    BertForQuestionAnswering,\n",
    "    BertTokenizer,\n",
    ")\n",
    "from transformers.data.metrics.squad_metrics import (\n",
    "    compute_predictions_logits,\n",
    "    squad_evaluate,\n",
    ")\n",
    "\n",
    "from transformers.data.processors.squad import SquadResult, SquadProcessor, squad_convert_examples_to_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "import config as cfg\n",
    "from utils import load_and_cache_examples, set_seed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "import apex\n",
    "from apex import amp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "mode = 'train'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-cased were not used when initializing BertForQuestionAnswering: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
      "- This IS expected if you are initializing BertForQuestionAnswering from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForQuestionAnswering from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForQuestionAnswering were not initialized from the model checkpoint at bert-base-cased and are newly initialized: ['qa_outputs.weight', 'qa_outputs.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "# BERT + 마지막 cls 추가 레이어 존재함\n",
    "# 추가 레이어는 학습이 되어있지 않으므로, 아래 Some weights of the model checkpoint at bert-large-cased were not used 와 같은 에러 발생\n",
    "# 추후 과제로 낼 시 이 부분을 각자 customize 하도록 과제를 내도 좋을듯 함\n",
    "model = BertForQuestionAnswering.from_pretrained(cfg.model_name)\n",
    "tokenizer = BertTokenizer.from_pretrained(cfg.tokenizer_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "model = model.to(cfg.device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/200 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating features from dataset file at %s ./data\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [00:09<00:00, 20.72it/s]\n",
      "convert squad examples to features: 100%|██████████| 38708/38708 [00:57<00:00, 668.00it/s] \n",
      "add example index and unique id: 100%|██████████| 38708/38708 [00:00<00:00, 1410098.66it/s]\n"
     ]
    }
   ],
   "source": [
    "train_dataset = load_and_cache_examples(cfg, tokenizer, mode_or_filename=mode, output_examples=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selected optimization level O1:  Insert automatic casts around Pytorch functions and Tensor methods.\n",
      "\n",
      "Defaults for this optimization level are:\n",
      "enabled                : True\n",
      "opt_level              : O1\n",
      "cast_model_type        : None\n",
      "patch_torch_functions  : True\n",
      "keep_batchnorm_fp32    : None\n",
      "master_weights         : None\n",
      "loss_scale             : dynamic\n",
      "Processing user overrides (additional kwargs that are not None)...\n",
      "After processing overrides, optimization options are:\n",
      "enabled                : True\n",
      "opt_level              : O1\n",
      "cast_model_type        : None\n",
      "patch_torch_functions  : True\n",
      "keep_batchnorm_fp32    : None\n",
      "master_weights         : None\n",
      "loss_scale             : dynamic\n"
     ]
    }
   ],
   "source": [
    "optimizer = apex.optimizers.FusedLAMB(model.parameters(),\n",
    "                                lr = cfg.learning_rate,\n",
    "                                eps=cfg.epsilon,\n",
    "                                weight_decay=cfg.weight_decay,\n",
    "                                max_grad_norm=cfg.max_grad_norm)\n",
    "model, optimizer = amp.initialize(model, optimizer, opt_level=\"O1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|▌         | 1/20 [00:00<00:02,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating features from dataset file at %s ./data\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:01<00:00, 18.75it/s]\n",
      "convert squad examples to features: 100%|██████████| 4639/4639 [00:06<00:00, 693.77it/s]\n",
      "add example index and unique id: 100%|██████████| 4639/4639 [00:00<00:00, 1421180.06it/s]\n"
     ]
    }
   ],
   "source": [
    "dataset, examples, features = load_and_cache_examples(cfg, tokenizer, mode='dev', output_examples=True)\n",
    "eval_sampler = SequentialSampler(dataset)\n",
    "eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=cfg.eval_batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "def evaluate(model, tokenizer):\n",
    "    print(\"***** Running evaluation *****\")\n",
    "    print(\"  Num examples = \", len(dataset))\n",
    "    print(\"  Batch size = \", cfg.eval_batch_size)\n",
    "    all_results = []\n",
    "    for batch in tqdm(eval_dataloader, desc=\"Evaluating\"):\n",
    "        model.eval()\n",
    "        batch = tuple(t.to(cfg.device) for t in batch)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            inputs = {\n",
    "                \"input_ids\": batch[0],\n",
    "                \"attention_mask\": batch[1],\n",
    "                \"token_type_ids\": batch[2],\n",
    "            }\n",
    "\n",
    "            feature_indices = batch[3]\n",
    "            outputs = model(**inputs)\n",
    "\n",
    "        for i, feature_index in enumerate(feature_indices):\n",
    "            eval_feature = features[feature_index.item()]\n",
    "            unique_id = int(eval_feature.unique_id)\n",
    "\n",
    "            start_logits = outputs.start_logits[i]\n",
    "            end_logits = outputs.end_logits[i]\n",
    "            result = SquadResult(unique_id, start_logits, end_logits)\n",
    "\n",
    "            all_results.append(result)\n",
    "            \n",
    "    predictions = compute_predictions_logits(examples,\n",
    "                                            features,\n",
    "                                            all_results,\n",
    "                                            cfg.n_best_size,\n",
    "                                            cfg.max_answer_length,\n",
    "                                            True,\n",
    "                                            None,\n",
    "                                            None,\n",
    "                                            None,\n",
    "                                            cfg.verbose_logging,\n",
    "                                            False,\n",
    "                                            cfg.null_score_diff_threshold,\n",
    "                                            tokenizer,)\n",
    "    results = squad_evaluate(examples, predictions)\n",
    "    return results\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "train_sampler = RandomSampler(train_dataset)\n",
    "train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=cfg.train_batch_size)\n",
    "\n",
    "t_total = len(train_dataloader) // cfg.gradient_accumulation_steps * cfg.num_train_epochs\n",
    "\n",
    "global_step = 1\n",
    "tr_loss = 0.0\n",
    "best_metrics = {'f1': 0, 'exact': 0, 'epoch': -1}\n",
    "model.zero_grad()\n",
    "# Added here for reproductibility\n",
    "set_seed(cfg.seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***** Running training *****\n",
      "  Num examples =  39189\n",
      "  Num Epochs =  5.0\n",
      "  Total train batch size =  16\n",
      "  Gradient Accumulation steps =  2\n",
      "  Total optimization steps =  12245.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e25e1b54a0942b491a38f108294bf76",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch::   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4c642885e9b4a26a970386f8ce2a5cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration::   0%|          | 0/4899 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 32768.0\n",
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 16384.0\n",
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 8192.0\n",
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 4096.0\n",
      "***** Running evaluation *****\n",
      "  Num examples = %d 4654\n",
      "  Batch size = %d 8\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb45ea914a5847b1bf3643658f1e7a08",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating:   0%|          | 0/582 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dev eval_exact: 66.99719767191205\n",
      "dev eval_f1: 79.9423909806456\n",
      "dev eval_total: 4639\n",
      "dev eval_HasAns_exact: 66.99719767191205\n",
      "dev eval_HasAns_f1: 79.9423909806456\n",
      "dev eval_HasAns_total: 4639\n",
      "dev eval_best_exact: 66.99719767191205\n",
      "dev eval_best_exact_thresh: 0.0\n",
      "dev eval_best_f1: 79.9423909806456\n",
      "dev eval_best_f1_thresh: 0.0\n",
      "dev best eval_f1: 79.9423909806456\n",
      "dev best eval_exact: 66.99719767191205\n",
      "dev best eval_epoch: 0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a041f47c2434999add120964bdad8fd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration::   0%|          | 0/4899 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 8192.0\n",
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 4096.0\n",
      "***** Running evaluation *****\n",
      "  Num examples = %d 4654\n",
      "  Batch size = %d 8\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3297dd0ab01419a89431c2efeddf767",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating:   0%|          | 0/582 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dev eval_exact: 70.29532226773011\n",
      "dev eval_f1: 82.54096893361755\n",
      "dev eval_total: 4639\n",
      "dev eval_HasAns_exact: 70.29532226773011\n",
      "dev eval_HasAns_f1: 82.54096893361755\n",
      "dev eval_HasAns_total: 4639\n",
      "dev eval_best_exact: 70.29532226773011\n",
      "dev eval_best_exact_thresh: 0.0\n",
      "dev eval_best_f1: 82.54096893361755\n",
      "dev eval_best_f1_thresh: 0.0\n",
      "dev best eval_f1: 82.54096893361755\n",
      "dev best eval_exact: 70.29532226773011\n",
      "dev best eval_epoch: 1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fcbe6f48730147bf92cb218934c23e8d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration::   0%|          | 0/4899 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 4096.0\n",
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 4096.0\n",
      "***** Running evaluation *****\n",
      "  Num examples = %d 4654\n",
      "  Batch size = %d 8\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd20b6f9cf6743949480160ab2a6821a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating:   0%|          | 0/582 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dev eval_exact: 70.61866781633972\n",
      "dev eval_f1: 82.4030336016294\n",
      "dev eval_total: 4639\n",
      "dev eval_HasAns_exact: 70.61866781633972\n",
      "dev eval_HasAns_f1: 82.4030336016294\n",
      "dev eval_HasAns_total: 4639\n",
      "dev eval_best_exact: 70.61866781633972\n",
      "dev eval_best_exact_thresh: 0.0\n",
      "dev eval_best_f1: 82.4030336016294\n",
      "dev eval_best_f1_thresh: 0.0\n",
      "dev best eval_f1: 82.54096893361755\n",
      "dev best eval_exact: 70.29532226773011\n",
      "dev best eval_epoch: 1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "51fe65a845134cdaa4f028812133d5a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration::   0%|          | 0/4899 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 2048.0\n",
      "***** Running evaluation *****\n",
      "  Num examples = %d 4654\n",
      "  Batch size = %d 8\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b2304218c7f4fbc8f3b093f4c5a38cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating:   0%|          | 0/582 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dev eval_exact: 71.07135158439318\n",
      "dev eval_f1: 82.74566015061633\n",
      "dev eval_total: 4639\n",
      "dev eval_HasAns_exact: 71.07135158439318\n",
      "dev eval_HasAns_f1: 82.74566015061633\n",
      "dev eval_HasAns_total: 4639\n",
      "dev eval_best_exact: 71.07135158439318\n",
      "dev eval_best_exact_thresh: 0.0\n",
      "dev eval_best_f1: 82.74566015061633\n",
      "dev eval_best_f1_thresh: 0.0\n",
      "dev best eval_f1: 82.74566015061633\n",
      "dev best eval_exact: 71.07135158439318\n",
      "dev best eval_epoch: 3\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7ff61422018f4115b75ffb2d5f9fadb0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration::   0%|          | 0/4899 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 8192.0\n",
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 4096.0\n",
      "***** Running evaluation *****\n",
      "  Num examples = %d 4654\n",
      "  Batch size = %d 8\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0a60979c2f424f20a694416036ee5304",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating:   0%|          | 0/582 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dev eval_exact: 70.27376589782281\n",
      "dev eval_f1: 82.27344184912754\n",
      "dev eval_total: 4639\n",
      "dev eval_HasAns_exact: 70.27376589782281\n",
      "dev eval_HasAns_f1: 82.27344184912754\n",
      "dev eval_HasAns_total: 4639\n",
      "dev eval_best_exact: 70.27376589782281\n",
      "dev eval_best_exact_thresh: 0.0\n",
      "dev eval_best_f1: 82.27344184912754\n",
      "dev eval_best_f1_thresh: 0.0\n",
      "dev best eval_f1: 82.74566015061633\n",
      "dev best eval_exact: 71.07135158439318\n",
      "dev best eval_epoch: 3\n"
     ]
    }
   ],
   "source": [
    "# Train!\n",
    "print(\"***** Running training *****\")\n",
    "print(\"  Num examples = \", len(train_dataset))\n",
    "print(\"  Num Epochs = \", cfg.num_train_epochs)\n",
    "print(\n",
    "    \"  Total train batch size = \",\n",
    "    cfg.train_batch_size\n",
    "    * cfg.gradient_accumulation_steps\n",
    ")\n",
    "print(\"  Gradient Accumulation steps = \", cfg.gradient_accumulation_steps)\n",
    "print(\"  Total optimization steps = \", t_total)\n",
    "\n",
    "for now_epoch in trange(int(cfg.num_train_epochs), desc=\"Epoch:\"):\n",
    "\n",
    "    for step, batch in enumerate(tqdm(train_dataloader, desc=\"Iteration:\")):\n",
    "        model.train()\n",
    "        batch = tuple(t.to(cfg.device) for t in batch)\n",
    "                                 \n",
    "        inputs = {\n",
    "            \"input_ids\": batch[0],\n",
    "            \"attention_mask\": batch[1],\n",
    "            \"token_type_ids\": batch[2],\n",
    "            \"start_positions\": batch[3],\n",
    "            \"end_positions\": batch[4],\n",
    "        }\n",
    "\n",
    "        outputs = model(**inputs)\n",
    "        loss = outputs[0]\n",
    "\n",
    "        if cfg.gradient_accumulation_steps > 1:\n",
    "            loss = loss / cfg.gradient_accumulation_steps\n",
    "\n",
    "        with amp.scale_loss(loss, optimizer) as scaled_loss:\n",
    "            scaled_loss.backward()\n",
    "\n",
    "        tr_loss += loss.item()\n",
    "        if (step + 1) % cfg.gradient_accumulation_steps == 0:\n",
    "            torch.nn.utils.clip_grad_norm_(model.parameters(), cfg.max_grad_norm)\n",
    "\n",
    "            optimizer.step()\n",
    "            model.zero_grad()\n",
    "            global_step += 1\n",
    "\n",
    "    results = evaluate(model, tokenizer)\n",
    "\n",
    "    if best_metrics['f1'] < results['f1']:\n",
    "        best_metrics['f1'] = results['f1']\n",
    "        best_metrics['exact'] = results['exact']\n",
    "        best_metrics['epoch'] = now_epoch\n",
    "        model.save_pretrained(cfg.output_dir)\n",
    "\n",
    "    for key, value in results.items():\n",
    "        print(\"dev eval_{}: {}\".format(key, value))\n",
    "\n",
    "    for key, value in best_metrics.items():\n",
    "        print(\"dev best eval_{}: {}\".format(key, value))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# 결과 20200120\n",
    "# 현재 환경 기준 한 epoch 이 도는 시간 약 7분\n",
    "# eval 수행하는 시간 약 16분\n",
    "# 그래서 한 epoch 당 23분 * 5 = 115 분 걸림"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}