convolutions2vit-cvt.ipynb

{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"vscode":{"interpreter":{"hash":"916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"}}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### CvT: Introducing Convolutions to Vision Transformers","metadata":{}},{"cell_type":"markdown","source":"### paper -> https://arxiv.org/pdf/2103.15808.pdf\n\nnotes: \n- Despite the success of vision Transformers at large scale,\nthe performance is still below similarly sized convolutional\nneural network (CNN) counterparts (e.g., ResNets)\nwhen trained on smaller amounts of data\n- images have a strong 2D local structure: spatially neighbor-\ning pixels are usually highly correlated\n\n- Instead of conv, they use depthwise separable convolution which reduce complexity. -> https://arxiv.org/pdf/1610.02357.pdf\n- They use stride 2 for key and value, which reduce the complexity for 4 times. But this stiation leads to decrase acc for deit\n- Conv stores location information, so they dropped the positional embedding.","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision\nimport torch.nn.functional as F\n!pip install einops\nimport einops\nimport torchvision\nimport torchvision.transforms as transforms\nimport cv2\nimport numpy as np\nfrom torchvision import datasets\nfrom torch.utils.data.sampler import SubsetRandomSampler\nimport os\nfrom pathlib import Path\nimport torch\nfrom PIL import Image\nfrom tqdm import tqdm\nfrom torchmetrics import F1Score,Accuracy\nimport matplotlib.pyplot as plt\nimport copy\nimport pytorch_lightning as pl\nimport pandas as pd\nimport seaborn as sn\nfrom IPython.core.display import display\nfrom torch.utils.data import random_split, DataLoader\n","metadata":{"execution":{"iopub.status.busy":"2022-10-22T20:26:56.591929Z","iopub.execute_input":"2022-10-22T20:26:56.593039Z","iopub.status.idle":"2022-10-22T20:27:20.357934Z","shell.execute_reply.started":"2022-10-22T20:26:56.592916Z","shell.execute_reply":"2022-10-22T20:27:20.356820Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"Collecting einops\n  Downloading einops-0.5.0-py3-none-any.whl (36 kB)\nInstalling collected packages: einops\nSuccessfully installed einops-0.5.0\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m","output_type":"stream"}]},{"cell_type":"code","source":"im_size = 224\nim_channel = 3\nembed_kernel_1 = 7\nembed_kernel_2 = 3\nembed_kernel_3 = 3\nstride_1 = 4\nstride_2 = 2\nstride_3 = 2\nout_channel_1 = 64\nout_channel_2 = 192\nout_channel_3 = 384\nproj_kernels = 3\nnum_heads_1 = 1\nnum_heads_2 = 4\nnum_heads_3 = 6\nbatch_size = 8 \nstage_1_N = 1\nstage_2_N = 2\nstage_3_N = 9 #+1 defined externally \nnum_class = 13\ndevice = \"cuda\"\nepochs = 2","metadata":{"execution":{"iopub.status.busy":"2022-10-22T20:27:20.359901Z","iopub.execute_input":"2022-10-22T20:27:20.360637Z","iopub.status.idle":"2022-10-22T20:27:20.368035Z","shell.execute_reply.started":"2022-10-22T20:27:20.360595Z","shell.execute_reply":"2022-10-22T20:27:20.366840Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"class DotProductAttention(nn.Module):\n    def __init__(self,):\n        super().__init__()\n        self.softmax = nn.Softmax(dim = 1)\n    def forward(self,query,key,value):\n        sdp = torch.matmul(self.softmax(torch.matmul(query,key)/key.size(dim = 2)**(1/2)),value)\n        return sdp","metadata":{"execution":{"iopub.status.busy":"2022-10-22T20:27:20.369482Z","iopub.execute_input":"2022-10-22T20:27:20.370381Z","iopub.status.idle":"2022-10-22T20:27:20.379727Z","shell.execute_reply.started":"2022-10-22T20:27:20.370343Z","shell.execute_reply":"2022-10-22T20:27:20.378887Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"code","source":"class ConvolutionalTokenEmbedding(nn.Module):\n    def __init__(self,in_channels:int,kernel_size:int, stride:int,batch_size:int,out_channels:int):\n        super().__init__()\n        padding = kernel_size//2\n        self.batch_size = batch_size\n        self.out_channels = out_channels\n        self.conv = nn.Conv2d(in_channels=in_channels,out_channels=out_channels,kernel_size= kernel_size,padding=padding,stride = stride)\n    def forward(self,img:torch.Tensor):\n        out = self.conv(img)\n        out = torch.reshape(out,(self.batch_size,self.out_channels,-1))\n        out = F.layer_norm(out,(out.shape[2],))\n        return out\n        ","metadata":{"execution":{"iopub.status.busy":"2022-10-22T20:27:20.382380Z","iopub.execute_input":"2022-10-22T20:27:20.382879Z","iopub.status.idle":"2022-10-22T20:27:20.393973Z","shell.execute_reply.started":"2022-10-22T20:27:20.382842Z","shell.execute_reply":"2022-10-22T20:27:20.392899Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"class depthwise_separable_conv(nn.Module): \n    def __init__(self, in_channels:int, kernels_per_layer:int,out_chanels:int,stride:int,):\n        super(depthwise_separable_conv, self).__init__()\n        \n        self.depthwise = nn.Conv2d(in_channels, in_channels * kernels_per_layer, kernel_size=3, padding=1,stride=stride ,groups=in_channels)\n        self.pointwise = nn.Conv2d(in_channels * kernels_per_layer, out_chanels, kernel_size=1)\n        self.batch_norm = nn.BatchNorm2d(in_channels * kernels_per_layer)\n\n    def forward(self, embed:torch.Tensor):\n        b_size,depth,length = embed.shape\n        out = torch.reshape(embed,(b_size,depth,int(length**(1/2)),int(length**(1/2))))\n        out = self.depthwise(out)\n        out = self.batch_norm(out)\n        out = self.pointwise(out)\n        out = torch.reshape(out,(out.shape[0],out.shape[1],-1))\n\n        return out","metadata":{"execution":{"iopub.status.busy":"2022-10-22T20:27:20.395350Z","iopub.execute_input":"2022-10-22T20:27:20.396177Z","iopub.status.idle":"2022-10-22T20:27:20.406769Z","shell.execute_reply.started":"2022-10-22T20:27:20.396138Z","shell.execute_reply":"2022-10-22T20:27:20.405929Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"class ConvolutionalProjection(nn.Module):\n    def __init__(self,in_channels:int,kernels_per_layer:int):\n        super().__init__()\n        self.conv_q = depthwise_separable_conv(in_channels,kernels_per_layer,in_channels,1)\n        self.conv_k = depthwise_separable_conv(in_channels,kernels_per_layer,in_channels,2)\n        self.conv_v = depthwise_separable_conv(in_channels,kernels_per_layer,in_channels,2)\n    def forward(self,embed:torch.Tensor):\n        return self.conv_q(embed),self.conv_k(embed),self.conv_v(embed)\n        ","metadata":{"execution":{"iopub.status.busy":"2022-10-22T20:27:20.408150Z","iopub.execute_input":"2022-10-22T20:27:20.408694Z","iopub.status.idle":"2022-10-22T20:27:20.428261Z","shell.execute_reply.started":"2022-10-22T20:27:20.408642Z","shell.execute_reply":"2022-10-22T20:27:20.427428Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"code","source":"class DotProductAttention(nn.Module):\n    def __init__(self,):\n        super().__init__()\n        self.softmax = nn.Softmax(dim = 1)\n    def forward(self,query,key,value):\n        x = self.softmax(torch.matmul(key.transpose(2,3),query))\n        sdp = torch.matmul(self.softmax(torch.matmul(query.transpose(2,3),key)/key.size(dim = 2)**(1/2)),value.transpose(2,3))\n        return sdp","metadata":{"execution":{"iopub.status.busy":"2022-10-22T20:27:20.429822Z","iopub.execute_input":"2022-10-22T20:27:20.430168Z","iopub.status.idle":"2022-10-22T20:27:20.440323Z","shell.execute_reply.started":"2022-10-22T20:27:20.430133Z","shell.execute_reply":"2022-10-22T20:27:20.439514Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"code","source":"class MultiHeadAttention(nn.Module):\n    def __init__(self,embed_size_q,embed_size_kv,num_heads):\n        super().__init__()\n        self.num_heads = num_heads\n        \n        self.embed_size_q = embed_size_q\n        self.embed_size_kv = embed_size_kv\n        self.DotProductAttention = DotProductAttention()\n\n        self.key = nn.Linear(embed_size_kv,embed_size_kv,bias = False)\n        self.query = nn.Linear(embed_size_q,embed_size_q,bias = False)\n        self.value = nn.Linear(embed_size_kv,embed_size_kv,bias = False)\n\n        self.linear = nn.Linear(embed_size_q,embed_size_q,bias = False)  \n      \n\n    def forward(self,q,k,v):\n        batch_size = q.size(0)\n        query = self.query(q).view(batch_size,-1,self.num_heads,self.embed_size_q)\n        key = self.key(k).view(batch_size,-1,self.num_heads,self.embed_size_kv)\n        value = self.value(v).view(batch_size,-1,self.num_heads,self.embed_size_kv)\n\n        query,key,value = query.transpose(1,2), key.transpose(1,2), value.transpose(1,2)\n        sdp = self.DotProductAttention(query,key,value)\n        return self.linear(sdp.view(batch_size,-1,self.embed_size_q))\n        \n","metadata":{"execution":{"iopub.status.busy":"2022-10-22T20:27:20.443510Z","iopub.execute_input":"2022-10-22T20:27:20.443860Z","iopub.status.idle":"2022-10-22T20:27:20.456799Z","shell.execute_reply.started":"2022-10-22T20:27:20.443832Z","shell.execute_reply":"2022-10-22T20:27:20.455929Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"code","source":"class ConvTransformerBlock(nn.Module):\n    def __init__(self,in_channels,kernels_per_layer,embed_size_q,embed_size_kv,num_heads ):\n        super().__init__()\n        \n        self.mha = MultiHeadAttention(embed_size_q,embed_size_kv,num_heads)\n        self.conv_proj = ConvolutionalProjection(in_channels,kernels_per_layer)\n        self.Linear1 = nn.Linear(embed_size_q,embed_size_q*4,bias=False)\n        self.Linear2 = nn.Linear(embed_size_q*4,embed_size_q,bias = False)\n        self.gelu = nn.GELU()\n        self.layer_norm = nn.LayerNorm(embed_size_q,eps = 1e-6)\n        \n\n    def forward(self,embed):\n        q,k,v = self.conv_proj(embed)\n        mha  = self.mha(q,k,v)\n        embed =  embed + self.mha(q,k,v)\n        layer_normed = self.layer_norm(embed)\n        linear1 = self.Linear1(layer_normed)\n        linear2 = self.Linear2(linear1)\n        embed = embed+ self.Linear2(self.gelu(self.Linear1(self.layer_norm(embed))))\n        return embed\n","metadata":{"execution":{"iopub.status.busy":"2022-10-22T20:27:20.459988Z","iopub.execute_input":"2022-10-22T20:27:20.460251Z","iopub.status.idle":"2022-10-22T20:27:20.475488Z","shell.execute_reply.started":"2022-10-22T20:27:20.460226Z","shell.execute_reply":"2022-10-22T20:27:20.474695Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"code","source":"class Stage(nn.Module):\n    def __init__(self,in_channels,kernels_per_layer, out_channels,embed_size_q,embed_size_kv,num_heads,kernel_size,stride,batch_size,N,last = False):\n        super().__init__()\n        self.batch_size = batch_size\n        self.in_channels = in_channels\n        self.out_channels = out_channels\n        self.last = last\n        self.conv_token_embed = ConvolutionalTokenEmbedding(in_channels,kernel_size,stride,batch_size,out_channels)\n\n        self.tiny_block = [ConvTransformerBlock(out_channels,kernels_per_layer,embed_size_q,embed_size_kv,num_heads) for i in range(N)]\n        self.block_seq = nn.Sequential(*self.tiny_block)\n\n\n    def forward(self,img:torch.Tensor):\n        if len(img.shape) == 3:\n            img = img.view(self.batch_size,self.in_channels,int(img.shape[-1]**(1/2)),int(img.shape[-1]**(1/2)))\n        out = self.conv_token_embed(img)\n        out = self.block_seq(out)\n        return out\n","metadata":{"execution":{"iopub.status.busy":"2022-10-22T20:27:20.479906Z","iopub.execute_input":"2022-10-22T20:27:20.480205Z","iopub.status.idle":"2022-10-22T20:27:20.493083Z","shell.execute_reply.started":"2022-10-22T20:27:20.480157Z","shell.execute_reply":"2022-10-22T20:27:20.492283Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self,num_class,im_size,im_channel,embed_kernel_1,embed_kernel_2,embed_kernel_3,stride_1,stride_2,stride_3,out_channel_1,out_channel_2,out_channel_3,proj_kernels,num_heads_1,num_heads_2,num_heads_3,stage_1_N,stage_2_N,stage_3_N,batch_size):\n        super().__init__()\n\n        embed_size_q_1 = int((im_size/stride_1)**2)\n        embed_size_q_2 = int(embed_size_q_1/(stride_2**2))\n        embed_size_q_3 = int(embed_size_q_2/(stride_3**2))\n        kernels_per_layer = 6\n        self.out_channel_3 = out_channel_3\n        self.stage1 = Stage(im_channel,kernels_per_layer, out_channel_1,int(embed_size_q_1),int(embed_size_q_1/4),num_heads_1,embed_kernel_1,stride_1,batch_size,stage_1_N,last = False)\n        \n        self.stage2 = Stage(out_channel_1,kernels_per_layer, out_channel_2,int(embed_size_q_2),int(embed_size_q_2/4),num_heads_2,embed_kernel_2,stride_2,batch_size,stage_2_N,last = False)\n        \n        self.stage3 = Stage(out_channel_2,kernels_per_layer, out_channel_3,int(embed_size_q_3),int(embed_size_q_3/4),num_heads_3,embed_kernel_3,stride_3,batch_size,stage_3_N,last = False)\n        #class token added conv_transformer_block\n        self.cls_token = nn.Parameter(torch.randn(1,out_channel_3,1))\n\n        self.mha = MultiHeadAttention(embed_size_q_3+1,int(embed_size_q_3/4)+1,num_heads_3)\n        self.conv_proj = ConvolutionalProjection(out_channel_3,kernels_per_layer)\n        self.Linear1 = nn.Linear(embed_size_q_3+1,(embed_size_q_3+1)*4,bias=False)\n        self.Linear2 = nn.Linear((embed_size_q_3+1)*4,embed_size_q_3+1,bias = False)\n        self.gelu = nn.GELU()\n        self.layer_norm = nn.LayerNorm(embed_size_q_3+1,eps = 1e-6)\n\n        self.mlp = nn.Sequential(nn.Linear(out_channel_3, out_channel_2),nn.GELU(),nn.Linear(out_channel_2,out_channel_1),nn.GELU(),nn.Linear(out_channel_1,num_class),nn.LogSoftmax(dim = 1))\n\n    def forward(self,image):\n        out = self.stage1(image)\n        out = self.stage2(out)\n        out = self.stage3(out)\n        q,k,v = self.conv_proj(out)\n        cls_token = self.cls_token.expand(out.shape[0],-1,-1)\n        q , k , v= torch.cat((cls_token,q),dim = 2) , torch.cat((cls_token,k), dim = 2) , torch.cat((cls_token,v), dim = 2)\n        mha =  self.mha(q,k,v)\n        out = mha + self.Linear2(self.gelu(self.Linear1(self.layer_norm(mha))))\n        out = self.mlp(out[:,:,-1])\n        return out \n","metadata":{"execution":{"iopub.status.busy":"2022-10-22T20:28:04.607542Z","iopub.execute_input":"2022-10-22T20:28:04.607981Z","iopub.status.idle":"2022-10-22T20:28:04.625978Z","shell.execute_reply.started":"2022-10-22T20:28:04.607945Z","shell.execute_reply":"2022-10-22T20:28:04.624798Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"code","source":"class Custom_Dataset():\n\n    def __init__(self, directory):\n        self.path = Path(directory)\n        Path.ls = lambda x: list(x.iterdir())\n        try:\n            files = os.listdir(directory)\n        except:\n            print(\"wrong path\")\n        self.x = [torch.tensor(np.transpose(np.array(Image.open(img).resize((224,224)))[:, :, :3], (2, 0, 1))).type(\n            torch.FloatTensor) for img in (self.path/files[0]).ls()]\n        self.x = torch.stack(self.x)/255\n        self.y = torch.tensor([0]*len((self.path/files[0]).ls()))\n        \n        for i in range(len(files)-1):\n            try:\n                self.x2 = [torch.tensor(np.transpose(np.array(Image.open(img).resize((224,224)))[:, :, :3], (2, 0, 1))).type(\n                torch.FloatTensor) for img in (self.path/files[i+1]).ls()]\n            except:\n                return \n            self.x2 = torch.stack(self.x2)/255\n            self.x = torch.cat((self.x, self.x2), 0)\n            self.y = torch.cat((self.y, torch.tensor(\n                [i+1]*len((self.path/files[i+1]).ls()))))\n        \n    def __len__(self):\n        return len(self.x)\n    \n    def __getitem__(self, i):\n        return self.x[i], self.y[i]\n    ","metadata":{"execution":{"iopub.status.busy":"2022-10-22T20:28:05.435276Z","iopub.execute_input":"2022-10-22T20:28:05.435628Z","iopub.status.idle":"2022-10-22T20:28:05.448362Z","shell.execute_reply.started":"2022-10-22T20:28:05.435597Z","shell.execute_reply":"2022-10-22T20:28:05.447174Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"code","source":"dataset = Custom_Dataset(\"../input/animal-image-classification-dataset/Animal Image Dataset\")\ndata_train, data_val = random_split(dataset, [len(dataset)-3000, 3000])","metadata":{"execution":{"iopub.status.busy":"2022-10-22T20:28:07.853240Z","iopub.execute_input":"2022-10-22T20:28:07.853897Z","iopub.status.idle":"2022-10-22T20:29:37.489456Z","shell.execute_reply.started":"2022-10-22T20:28:07.853862Z","shell.execute_reply":"2022-10-22T20:29:37.488357Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"trainloader =DataLoader(data_train, batch_size=8,drop_last=True)\nvalidloader = DataLoader(data_val, batch_size=8,drop_last=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-22T20:29:37.491520Z","iopub.execute_input":"2022-10-22T20:29:37.491922Z","iopub.status.idle":"2022-10-22T20:29:37.497612Z","shell.execute_reply.started":"2022-10-22T20:29:37.491885Z","shell.execute_reply":"2022-10-22T20:29:37.496707Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"code","source":"f1 = F1Score().to(device)","metadata":{"execution":{"iopub.status.busy":"2022-10-22T20:30:15.500147Z","iopub.execute_input":"2022-10-22T20:30:15.500520Z","iopub.status.idle":"2022-10-22T20:30:20.765410Z","shell.execute_reply.started":"2022-10-22T20:30:15.500486Z","shell.execute_reply":"2022-10-22T20:30:20.764383Z"},"trusted":true},"execution_count":16,"outputs":[]},{"cell_type":"code","source":"criterion = nn.NLLLoss()#LabelSmoothing_NLLL(smoothing =label_smooth )","metadata":{"execution":{"iopub.status.busy":"2022-10-22T20:30:20.767223Z","iopub.execute_input":"2022-10-22T20:30:20.767612Z","iopub.status.idle":"2022-10-22T20:30:20.774064Z","shell.execute_reply.started":"2022-10-22T20:30:20.767575Z","shell.execute_reply":"2022-10-22T20:30:20.772727Z"},"trusted":true},"execution_count":17,"outputs":[]},{"cell_type":"code","source":"model = Model(num_class,im_size,im_channel,embed_kernel_1,embed_kernel_2,embed_kernel_3,stride_1,stride_2,stride_3,out_channel_1,out_channel_2,out_channel_3,proj_kernels,num_heads_1,num_heads_2,num_heads_3,stage_1_N,stage_2_N,stage_3_N,batch_size).to(device)","metadata":{"execution":{"iopub.status.busy":"2022-10-22T20:30:22.161251Z","iopub.execute_input":"2022-10-22T20:30:22.161601Z","iopub.status.idle":"2022-10-22T20:30:23.743776Z","shell.execute_reply.started":"2022-10-22T20:30:22.161571Z","shell.execute_reply":"2022-10-22T20:30:23.742782Z"},"trusted":true},"execution_count":18,"outputs":[]},{"cell_type":"code","source":"learning_rate = 0.0001\nbeta1 = 0.9\nbeta2 = 0.990\nweight_decay = 0.3\noptimizer = torch.optim.Adam(model.parameters(),lr = learning_rate,betas = (beta1,beta2),weight_decay=weight_decay)","metadata":{"execution":{"iopub.status.busy":"2022-10-22T20:30:24.890620Z","iopub.execute_input":"2022-10-22T20:30:24.894007Z","iopub.status.idle":"2022-10-22T20:30:24.907082Z","shell.execute_reply.started":"2022-10-22T20:30:24.893953Z","shell.execute_reply":"2022-10-22T20:30:24.905970Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"code","source":"device = \"cuda\"\nvalid_loss_min = np.Inf\nfor i in range(20):\n    print(\"Epoch - {} Started\".format(i+1))\n\n    train_loss = 0.0\n    valid_loss = 0.0\n    train_score = 0.0\n    val_score = 0.0\n    model.train()\n    for data, target in trainloader:\n        data, target = data.to(device), target.to(device)\n        optimizer.zero_grad()\n        output = model(data)\n        loss = criterion(output, target)\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item()*data.size(0)\n        train_score = train_score +f1(output,target)\n    \n    model.eval()\n    \n    for data, target in validloader:\n        data, target = data.to(device), target.to(device)\n        with torch.no_grad():\n            output = model(data)\n        loss = criterion(output, target)\n        valid_loss += loss.item()*data.size(0)\n        val_score = val_score + f1(output,target)\n\n    \n    train_loss = train_loss/len(trainloader.sampler)\n    valid_loss = valid_loss/len(validloader.sampler)\n    train_score = batch_size*train_score/len(trainloader.sampler)\n    val_score = batch_size*val_score/len(validloader.sampler)\n    \n    \n    \n    \n    print(f\"F1 Score for train: {train_score}, F1 Score for validation: {val_score} \")\n    print('Epoch: {} \\tTraining Loss: {:.6f} \\tValidation Loss: {:.6f}'.format(\n        i+1, train_loss, valid_loss))\n\n    if valid_loss <= valid_loss_min:\n        print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(\n            valid_loss_min,\n            valid_loss))\n        best_model_wts = copy.deepcopy(model.state_dict())\n\n        valid_loss_min = valid_loss\ntorch.save(best_model_wts, 'model.pt')","metadata":{"execution":{"iopub.status.busy":"2022-10-22T20:30:25.998559Z","iopub.execute_input":"2022-10-22T20:30:25.999868Z"},"trusted":true},"execution_count":null,"outputs":[{"name":"stdout","text":"Epoch - 1 Started\nF1 Score for train: 0.25071224570274353, F1 Score for validation: 0.2516666650772095 \nEpoch: 1 \tTraining Loss: 1.631188 \tValidation Loss: 1.409681\nValidation loss decreased (inf --> 1.409681).  Saving model ...\nEpoch - 2 Started\nF1 Score for train: 0.2560541331768036, F1 Score for validation: 0.2523333430290222 \nEpoch: 2 \tTraining Loss: 1.414844 \tValidation Loss: 1.402783\nValidation loss decreased (1.409681 --> 1.402783).  Saving model ...\nEpoch - 3 Started\nF1 Score for train: 0.24750712513923645, F1 Score for validation: 0.24966666102409363 \nEpoch: 3 \tTraining Loss: 1.406472 \tValidation Loss: 1.400468\nValidation loss decreased (1.402783 --> 1.400468).  Saving model ...\nEpoch - 4 Started\nF1 Score for train: 0.24537037312984467, F1 Score for validation: 0.23633332550525665 \nEpoch: 4 \tTraining Loss: 1.404384 \tValidation Loss: 1.402872\nEpoch - 5 Started\nF1 Score for train: 0.2510683834552765, F1 Score for validation: 0.2516666650772095 \nEpoch: 5 \tTraining Loss: 1.403609 \tValidation Loss: 1.403174\nEpoch - 6 Started\nF1 Score for train: 0.24572649598121643, F1 Score for validation: 0.2516666650772095 \nEpoch: 6 \tTraining Loss: 1.409061 \tValidation Loss: 1.401930\nEpoch - 7 Started\nF1 Score for train: 0.24786324799060822, F1 Score for validation: 0.2516666650772095 \nEpoch: 7 \tTraining Loss: 1.401879 \tValidation Loss: 1.403138\nEpoch - 8 Started\nF1 Score for train: 0.24537037312984467, F1 Score for validation: 0.2516666650772095 \nEpoch: 8 \tTraining Loss: 1.401283 \tValidation Loss: 1.398874\nValidation loss decreased (1.400468 --> 1.398874).  Saving model ...\nEpoch - 9 Started\nF1 Score for train: 0.2439458668231964, F1 Score for validation: 0.2516666650772095 \nEpoch: 9 \tTraining Loss: 1.401215 \tValidation Loss: 1.397434\nValidation loss decreased (1.398874 --> 1.397434).  Saving model ...\nEpoch - 10 Started\nF1 Score for train: 0.2389601171016693, F1 Score for validation: 0.2516666650772095 \nEpoch: 10 \tTraining Loss: 1.401095 \tValidation Loss: 1.402363\nEpoch - 11 Started\nF1 Score for train: 0.2410968691110611, F1 Score for validation: 0.2516666650772095 \nEpoch: 11 \tTraining Loss: 1.400754 \tValidation Loss: 1.406651\nEpoch - 12 Started\nF1 Score for train: 0.24287749826908112, F1 Score for validation: 0.2516666650772095 \nEpoch: 12 \tTraining Loss: 1.400952 \tValidation Loss: 1.398515\nEpoch - 13 Started\nF1 Score for train: 0.24537037312984467, F1 Score for validation: 0.2516666650772095 \nEpoch: 13 \tTraining Loss: 1.402195 \tValidation Loss: 1.398902\nEpoch - 14 Started\nF1 Score for train: 0.2421652376651764, F1 Score for validation: 0.2516666650772095 \nEpoch: 14 \tTraining Loss: 1.400805 \tValidation Loss: 1.398346\nEpoch - 15 Started\n","output_type":"stream"}]}]}