Birds400 - Efficient Net - 매개변수 조정 및 정리

Library

import numpy as np
import matplotlib.pyplot as plt
import sys
import os
import time
import random 
import pandas as pd
import torch
from torch import nn, cuda, optim
from torchvision import models,transforms,datasets
from torch.utils.data import DataLoader,random_split
from PIL import Image
import seaborn as sns
import torch.nn.functional as F
import shutil
import torchvision
import torchvision.transforms as transforms
import torch
import torch.nn as nn
import torch.optim as optim

device = 'cuda' if cuda.is_available() else 'cpu'

1st submission

data argmentation

train_transform = transforms.Compose([transforms.RandomRotation(45),transforms.RandomHorizontalFlip(),
                                      transforms.RandomVerticalFlip(),transforms.ToTensor(),
                                      transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])
val_transform = transforms.Compose([transforms.RandomRotation(45),transforms.RandomHorizontalFlip(),
                                    transforms.RandomVerticalFlip(),transforms.ToTensor(),
                                    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])

data size

• train : 0.95 / valid : 0.05

data = datasets.ImageFolder(data_dir)
train_size = int(len(data)*0.95)
val_size = int((len(data)-train_size))
train_data,val_data = random_split(data,[train_size,val_size])
torch.manual_seed(3334)
print(f'train size: {len(train_data)}\nval size: {len(val_data)}')

parameter

• depth coeff : 1
• width coeff : 1

model = EfficientNet(1, 1, num_classes=400)

• optimizer = optim.Adam(model.parameters(), lr=0.0002, weight_decay=0.0005)
• scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones = [20, 40, 45], gamma = 0.5)
• batch size = 16

result

• accuracy : 0.95400

2nd submission

수정사항

• 이미지 분석 후 증강 과정에서 vertical flip 제거
• depth coeff를 1에서 1.06으로 늘려 제한사항인 5M미만의 최대 매개변수 설정
• Learning rage 0.0001에서 0.0002로 수정
• optimizer를 SGD와 Adam, NAdam의 결과 분석 후 Adam으로 선정
• scheduler를 5단계로 분할, gamma = 0.5

optimizer = optim.Adam(model.parameters(), lr=0.0002, weight_decay=0.0005)
# optimizer = torch.optim.SGD(model.parameters(), lr=0.02, momentum=0.9, nesterov=True, weight_decay=5e-4)
scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones = [25, 30, 40, 43, 47], gamma = 0.5)

result

• accuracy : 0.95450

3rd submission

수정사항

• 데이터 증강과정에서 이미지 회전을 45도에서 30도로 수정, vertical flip 삭제

train_transform = transforms.Compose([transforms.RandomRotation(30),transforms.RandomHorizontalFlip(),
                                      transforms.ToTensor(),transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])

• 훈련의 마지막 단계에서 overfitting문제 발생
  • dropout_rate 수정

class EfficientNet(nn.Module):
    def __init__(self, width_coeff, depth_coeff,
                 depth_div=8, min_depth=None,
                 dropout_rate=0.2, drop_connect_rate=0.2,
                 num_classes=400):
        super().__init__()
        min_depth = min_depth or depth_div
        
        def renew_ch(x):
            if not width_coeff:
                return x

            x *= width_coeff
            new_x = max(min_depth, int(x + depth_div / 2) // depth_div * depth_div)
            if new_x < 0.9 * x:
                new_x += depth_div
            return int(new_x)

        def renew_repeat(x):
            return int(math.ceil(x * depth_coeff))

        self.stem = conv_bn_act(3, renew_ch(32), kernel_size=3, stride=2, bias=False)
        
        self.blocks = nn.Sequential(
            #       input channel  output    expand  k  s                   skip  se
            MBBlock(renew_ch(32), renew_ch(16), 1, 3, 1, renew_repeat(1), True, 0.25, drop_connect_rate),
            MBBlock(renew_ch(16), renew_ch(24), 6, 3, 2, renew_repeat(2), True, 0.25, drop_connect_rate),
            MBBlock(renew_ch(24), renew_ch(40), 6, 5, 2, renew_repeat(2), True, 0.25, drop_connect_rate),
            MBBlock(renew_ch(40), renew_ch(80), 6, 3, 2, renew_repeat(3), True, 0.25, drop_connect_rate),
            MBBlock(renew_ch(80), renew_ch(112), 6, 5, 1, renew_repeat(3), True, 0.25, drop_connect_rate),
            MBBlock(renew_ch(112), renew_ch(192), 6, 5, 2, renew_repeat(4), True, 0.25, drop_connect_rate),
            MBBlock(renew_ch(192), renew_ch(320), 6, 3, 1, renew_repeat(1), True, 0.25, drop_connect_rate)
        )

        self.head = nn.Sequential(
            *conv_bn_act(renew_ch(320), renew_ch(1280), kernel_size=1, bias=False),
            nn.AdaptiveAvgPool2d(1),
            nn.Dropout2d(dropout_rate, True) if dropout_rate > 0 else nn.Identity(),
            Flatten(),
            nn.Linear(renew_ch(1280), num_classes)
        )

        self.init_weights()

    def init_weights(self):
        for m in self.modules():
            if isinstance(m, SamePadConv2d):
                nn.init.kaiming_normal_(m.weight, mode="fan_out")
            elif isinstance(m, nn.Linear):
                init_range = 1.0 / math.sqrt(m.weight.shape[1])
                nn.init.uniform_(m.weight, -init_range, init_range)

    def forward(self, inputs):
        stem = self.stem(inputs)
        x = self.blocks(stem)
        head = self.head(x)
        return head

result

• accuracy : 0.96150

4th submission

수정사항

• 데이터 증강과정에서 이미지회전을 30도에서 15도로 수정
• depth coeff 1로 수정
• dropout_rate 0으로 수정
• schduler 6단계로 분할
• optimizer SGD로 수정, Learning rate = 0.02로 수정

# optimizer = optim.Adam(model.parameters(), lr=0.0002, weight_decay=0.0005)
optimizer = torch.optim.SGD(model.parameters(), lr=0.02, momentum=0.9, nesterov=True, weight_decay=5e-4)
scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones = [20, 30, 35, 40, 43, 47], gamma = 0.5)

result

• accuracy : 0.97250

5th submission

• dropout rate, drop_connect_rate 각각 0으로 수정
• depth_coeff 1.06로 재조정

result

• accuracy : 0.98500