Add files via upload

caffe_cifar10.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Sun Nov  4 13:38:21 2018
+
+@author: seukgyo
+"""
+
+import torch.nn as nn
+
+class CIFAR10_QUICK(nn.Module):
+    def __init__(self):
+        super(CIFAR10_QUICK, self).__init__()
+
+        self.conv1 = nn.Conv2d(3, 32, 5, 1, 2)
+        self.pool1 = nn.MaxPool2d(3, 2)
+        self.relu1 = nn.ReLU()
+
+        self.conv2 = nn.Conv2d(32, 32, 5, 1, 2)
+        self.relu2 = nn.ReLU()
+        self.pool2 = nn.AvgPool2d(3, 2)
+
+        self.conv3 = nn.Conv2d(32, 64, 5, 1, 2)
+        self.relu3 = nn.ReLU()
+        self.pool3 = nn.AvgPool2d(3, 2)
+
+        self.ip1 = nn.Linear(576, 64)
+        self.ip2 = nn.Linear(64, 10)
+
+    def forward(self, img):
+        out = self.conv1(img)
+        out = self.pool1(out)
+        out = self.relu1(out)
+
+        out = self.conv2(out)
+        out = self.relu2(out)
+        out = self.pool2(out)
+
+        out = self.conv3(out)
+        out = self.relu3(out)
+        out = self.pool3(out)
+
+        out = out.view(-1, 576)
+
+        out = self.ip1(out)
+        out = self.ip2(out)
+
+        return out

choose_lr.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Sun Nov  4 13:36:42 2018
+
+@author: seukgyo
+"""
+
+import torch
+import torchvision
+import torchvision.transforms as transforms
+
+# The output of torchvision datasets are PILImage images of range [0, 1]
+# Transform them to Tensors of Normalized Range [-1, 1]
+
+transform = transforms.Compose(
+        [transforms.ToTensor(),
+         transforms.Normalize((0.5, 0.5, 0.5), (0.5,0.5,0.5))])
+
+trainset = torchvision.datasets.CIFAR10(root='./data', train=True,
+                                        download=True, transform=transform)
+
+trainloader = torch.utils.data.DataLoader(trainset, batch_size=100,
+                                          shuffle=True, num_workers=2)
+
+testset = torchvision.datasets.CIFAR10(root='./data', train=False,
+                                       download=True, transform=transform)
+
+testloader = torch.utils.data.DataLoader(testset, batch_size=100,
+                                         shuffle=False, num_workers=2)
+
+# Training on GPU
+
+device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
+
+print(device)
+
+# Define a Convolution Neural Network
+
+from caffe_cifar10 import CIFAR10_QUICK
+
+net = CIFAR10_QUICK()
+net = net.to(device)
+
+# Define a Loss Function
+
+import torch.nn as nn
+
+criterion = nn.CrossEntropyLoss()
+
+# Define Optimizer and Learning Rate
+
+import torch.optim as optim
+
+start_lr = 0.0
+end_lr = 0.02
+step = 0.001
+
+lr_step = int(end_lr/step)
+
+total_epoch = 8
+
+x_axis = []
+y_axis = []
+
+for i in range(lr_step+1):
+    lr = i * step
+
+    print('lr: %.4f' % (lr))
+
+    x_axis.append(lr)
+
+    optimizer = optim.SGD(net.parameters(), lr=lr, momentum=0.9)
+
+    # Train the network    
+    for epoch in range(total_epoch):
+        net.train()
+
+        running_loss = 0.0
+
+        for data in trainloader:
+            # get the inputs
+            inputs, labels = data
+
+            inputs = inputs.to(device)
+            labels = labels.to(device)
+
+            # zero the parameters gradients
+            optimizer.zero_grad()
+
+            # forward + backward + optimize
+            outputs = net(inputs)
+            loss = criterion(outputs, labels)
+            loss.backward()
+            optimizer.step()
+
+            running_loss += loss.item()
+
+        print('epoch: %d, loss: %.3f' % (epoch+1, running_loss))
+
+    print('Finished Training')
+
+    # Test the network on the test data
+    net.eval()
+
+    total = 0
+    correct = 0
+
+    for data in testloader:
+        images, labels = data
+
+        images = images.to(device)
+        labels = labels.to(device)
+
+        outputs = net(images)
+
+        _, pred = torch.max(outputs, 1)
+
+        total += labels.size(0)
+        correct += (pred == labels).sum().item()
+
+    accuracy = correct / total
+
+    y_axis.append(accuracy)
+    print('Accuracy : %.4f' % (accuracy))
+

clr.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Sun Nov  4 16:04:43 2018
+
+@author: seukgyo
+"""
+
+import numpy as np
+
+class CLR(object):
+    def __init__(self, optimizer, base_lr=0.001, max_lr=0.006, step_size=500.0,
+                 mode='triangular', gamma=1.0, scale_fn=None, 
+                 scale_mode='cycle', last_iteration=-1):
+        super(CLR, self).__init__()
+
+        self.optimizer = optimizer
+
+        self.base_lr = base_lr
+        self.max_lr = max_lr
+
+        self.step_size = step_size
+        self.mode = mode
+        self.gamma = gamma
+
+        if scale_fn == None:
+            if self.mode == 'triangular':
+                self.scale_fn = lambda x: 1.0
+                self.scale_mode = 'cycle'
+            elif self.mode == 'triangular2':
+                self.scale_fn = lambda x: 1/(2.0**(x-1))
+                self.scale_mode = 'cycle'
+            elif self.mode == 'exp_range':
+                self.scale_fn = lambda x: self.gamma**(x)
+                self.scale_mode = 'iterations'
+        else:
+            self.scale_fn = scale_fn
+            self.scale_mode = scale_mode
+
+        self.step(last_iteration+1)
+        self.last_iteration = last_iteration
+
+    def get_lr(self):
+        cycle = np.floor(1 + self.last_iteration / (2 * self.step_size))
+        x = np.abs(self.last_iteration/self.step_size - 2*cycle + 1)
+
+        base_height = (self.max_lr - self.base_lr) * np.maximum(0, (1-x))
+
+        if self.scale_mode == 'cycle':
+            lr = self.base_lr + base_height * self.scale_fn(cycle)
+        else:
+            lr = self.base_lr + base_height * self.scale_fn(self.last_iteration)
+
+        return lr
+
+    def step(self, batch_iteration=None):
+        if batch_iteration is None:
+            batch_iteration = self.last_iteration + 1
+
+        self.last_iteration = batch_iteration
+
+        lr = self.get_lr()
+        for param_group in self.optimizer.param_groups:
+            param_group['lr'] = lr
+

train_cifar10.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Nov 12 19:37:50 2018
+
+@author: seukgyo
+"""
+
+import torch
+import torchvision
+import torchvision.transforms as transforms
+
+# The output of torchvision datasets are PILImage images of range [0, 1]
+# Transform them to Tensors of Normalized Range [-1, 1]
+
+transform = transforms.Compose(
+        [transforms.ToTensor(),
+         transforms.Normalize((0.5, 0.5, 0.5), (0.5,0.5,0.5))])
+
+trainset = torchvision.datasets.CIFAR10(root='./data', train=True,
+                                        download=True, transform=transform)
+
+trainloader = torch.utils.data.DataLoader(trainset, batch_size=100,
+                                          shuffle=True, num_workers=2)
+
+testset = torchvision.datasets.CIFAR10(root='./data', train=False,
+                                       download=True, transform=transform)
+
+testloader = torch.utils.data.DataLoader(testset, batch_size=100,
+                                         shuffle=False, num_workers=2)
+
+# Training on GPU
+
+device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
+
+# Define a Convolution Neural Network
+
+from caffe_cifar10 import CIFAR10_QUICK
+
+net = CIFAR10_QUICK()
+net = net.to(device)
+
+# Define a Loss Function
+
+import torch.nn as nn
+
+criterion = nn.CrossEntropyLoss()
+
+import torch.optim as optim
+import clr
+
+total_epoch = 50
+
+optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)
+scheduler = clr.CLR(optimizer)
+
+# Train the network    
+for epoch in range(total_epoch):
+    net.train()
+
+    running_loss = 0.0
+
+    for data in trainloader:
+
+        scheduler.step()
+
+        # get the inputs
+        inputs, labels = data
+
+        inputs = inputs.to(device)
+        labels = labels.to(device)
+
+        # zero the parameters gradients
+        optimizer.zero_grad()
+
+        # forward + backward + optimize
+        outputs = net(inputs)
+        loss = criterion(outputs, labels)
+        loss.backward()
+        #%%
+        optimizer.step()
+
+        running_loss += loss.item()
+
+    print('epoch: %d, loss: %.3f' % (epoch+1, running_loss))
+
+print('Finished Training')
+
+# Test the network on the test data
+net.eval()
+
+total = 0
+correct = 0
+
+for data in testloader:
+    images, labels = data
+
+    images = images.to(device)
+    labels = labels.to(device)
+
+    outputs = net(images)
+
+    _, pred = torch.max(outputs, 1)
+
+    total += labels.size(0)
+    correct += (pred == labels).sum().item()
+
+accuracy = correct / total
+
+print('Accuracy : %.4f' % (accuracy))