pytorch_seresnext101-32x4d+kfold+focal loss v3.0

From: https://www.kaggle.com/tony92151/pytorch-seresnext101-32x4d-kfold-focal-loss-v3-0

Author: tonyguo

Score: 0.547

pytorch_seresnext101-32x4d+kfold+focal loss v3.0

  • fold = 4
  • epoch = 3
  • γ=0.4
In [1]:
!nvidia-smi
Thu Jun  6 05:43:35 2019       
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 410.104      Driver Version: 410.104      CUDA Version: 10.0     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  Tesla P100-PCIE...  On   | 00000000:00:04.0 Off |                    0 |
| N/A   44C    P0    29W / 250W |      0MiB / 16280MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+
                                                                               
+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU       PID   Type   Process name                             Usage      |
|=============================================================================|
|  No running processes found                                                 |
+-----------------------------------------------------------------------------+
In [2]:
import torch
from torch import nn, optim
import torch.nn.functional as F
import matplotlib.pyplot as plt
import sys
import time
import numpy as np
import math
import pandas as pd
from PIL import Image, ImageOps, ImageFilter
from datetime import datetime
from torch.autograd import Variable
from torch.utils.data import Dataset, DataLoader
import torchvision.transforms as transforms
from torchvision import datasets, models, transforms
import random
import datetime
import os

from sklearn import preprocessing 
from sklearn.model_selection import KFold

device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
print(device)
cuda:0
In [3]:
# os.chdir("../input/pretrained_PyTorch/")
# os.getcwd()
In [4]:
train = pd.read_csv("../input/imet-2019-fgvc6/train.csv")
lable = pd.read_csv("../input/imet-2019-fgvc6/labels.csv")
test = pd.read_csv("../input/imet-2019-fgvc6/sample_submission.csv")
In [5]:
lable_length = len(lable)
train_length = len(train)
test_length = len(test)
print(train_length)
print(lable_length)
print(test_length)
109237
1103
7443
In [6]:
print(np.array(lable)[397])
print(np.array(lable)[398])
c_length = len(np.array(lable)[:398])
t_length = len(np.array(lable)[398:])
print(c_length)
print(t_length)
[397 'culture::zurich']
[398 'tag::abbies']
398
705
In [7]:
#np.array(test)
In [8]:
test.head
Out[8]:
<bound method NDFrame.head of                     id attribute_ids
0     10023b2cc4ed5f68         0 1 2
1     100fbe75ed8fd887         0 1 2
2     101b627524a04f19         0 1 2
3     10234480c41284c6         0 1 2
4     1023b0e2636dcea8         0 1 2
5      1039cd6cf85845c         0 1 2
6      103a5b3f83fbe88         0 1 2
7     10413aaae8d6a9a2         0 1 2
8     10423822b93a65ab         0 1 2
9     1052bf702cb099f7         0 1 2
10     10543c918a43a8d         0 1 2
11    105c9a3453da79c3         0 1 2
12    1060688bbf6eac87         0 1 2
13    106a247caeabd15a         0 1 2
14    106e21606add59f3         0 1 2
15    107c38495881b6c9         0 1 2
16    108815dd3752ab64         0 1 2
17    10943defdd5d5e89         0 1 2
18    10a39a78c44ef27c         0 1 2
19     10ab70df067bdb4         0 1 2
20    10b28e3de3566582         0 1 2
21    10b32964331a6cc3         0 1 2
22    10b4562e7fa6f668         0 1 2
23    10db1c338e1d822f         0 1 2
24    10e0c215f5f3084e         0 1 2
25    10e95bead8e0b35b         0 1 2
26    1100d7b0f24fee88         0 1 2
27    11099b321e8c7066         0 1 2
28    110df388fd5c50e4         0 1 2
29    113520ea0138f76d         0 1 2
...                ...           ...
7413  ff2da1f0ed3e3ebe         0 1 2
7414  ff3a9fa43f8eab9c         0 1 2
7415  ff44490e20740a19         0 1 2
7416  ff481bb029678d5d         0 1 2
7417  ff4c3570fb7b90d3         0 1 2
7418  ff4f548d08414709         0 1 2
7419  ff668377a518ea5f         0 1 2
7420  ff6a549b2d7a0e76         0 1 2
7421  ff6ee1b37c8dc1ae         0 1 2
7422  ff85460d6b853b49         0 1 2
7423   ff8721b85d1b5a5         0 1 2
7424  ff8bef7d0de52b31         0 1 2
7425  ff92504c82c41e0f         0 1 2
7426  ff9d4b77c124c9f2         0 1 2
7427  ff9ddf70cb1c2674         0 1 2
7428  ffaf8c3fe0b1d9b6         0 1 2
7429  ffb61df4a6734772         0 1 2
7430  ffb73f95b8721900         0 1 2
7431  ffb937b55755323e         0 1 2
7432  ffbcf8b91a8e8ce0         0 1 2
7433  ffbf4849bde21b0a         0 1 2
7434  ffc96e053345419d         0 1 2
7435  ffcb16053099d795         0 1 2
7436  ffcf745289465074         0 1 2
7437  ffd1372fe67e65f0         0 1 2
7438  ffd79eadf642221b         0 1 2
7439  ffd96986aa333f4d         0 1 2
7440  ffe54b454396d97c         0 1 2
7441  ffe7d7db4e4aa37f         0 1 2
7442  ffed0a4aca0d5457         0 1 2

[7443 rows x 2 columns]>
In [9]:
#np.array(train)
In [10]:
def creatData(train,lable_length):
    train = np.array(train)
    trainA_data = []
    lab_data = []
    #trainC_data = []
    #trainT_data = []
    for t in range(train_length):
        v = np.zeros(lable_length)
        #print(train[t,1])
        lab = []
        for s in train[t,1].split(" "):
            #print(s)
            v[int(s)] = 1
            lab.append(int(s))
        trainA_data.append([train[t,0],v])
        lab_data.append([train[t,0],np.array(lab)])
        #trainC_data.append([train[t,0],v[:c_length]])
        #trainT_data.append([train[t,0],v[c_length:]])
    return np.array(trainA_data),np.array(lab_data)
    #return np.array(trainA_data)#,np.array(trainC_data),np.array(trainT_data)
In [11]:
train_a,train_lab = creatData(train,lable_length)
#train_a,train_c,train_t = creatData(train,lable_length)
#train_t.shape()
In [12]:
train_lab[0]
Out[12]:
array(['1000483014d91860', array([147, 616, 813])], dtype=object)

Remove the item that only have one feature (not work well)

In [13]:
def dfilter(data,th):
    dfilter = []
    for i in range(len(data)):
        if train_a[i][1].sum()>th:
            dfilter.append(train_a[i])
    return np.array(dfilter)
In [14]:
datafilter = dfilter(train_a,1) # remain feature great then 1
In [15]:
print(f"amount of image before: {len(train_a)}")

print(f"amount of image after: {len(datafilter)}")
amount of image before: 109237
amount of image after: 104913
In [16]:
# not use
datafilter = train_a
In [17]:
image_resize = 200
data_transforms2 = transforms.Compose([
    transforms.Resize((image_resize,image_resize)),
    #transforms.RandomResizedCrop(250),
    #transforms.RandomHorizontalFlip(),
    transforms.ToTensor(),
    transforms.Normalize(
            [0.485, 0.456, 0.406], 
            [0.229, 0.224, 0.225])
    ])


data_transforms = transforms.Compose([
    transforms.Resize((image_resize,image_resize)),
    #transforms.RandomResizedCrop(250),
    #transforms.RandomHorizontalFlip(),
    transforms.ToTensor(),
    #transforms.Normalize(
    #        [0.485, 0.456, 0.406], 
    #        [0.229, 0.224, 0.225])
    ])

train_transformer = transforms.Compose([
    transforms.Resize((128,128)),              # resize the image to 
    #transforms.RandomHorizontalFlip(),  # randomly flip image horizontally
    #transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
    transforms.ToTensor(),           # transform it into a PyTorch Tensor
    #transforms.Normalize(mean = (0.5, 0.5, 0.5), std = (0.5, 0.5, 0.5))
])
In [18]:
class trainDataset(Dataset):
    def __init__(self, train_lib, transform,transform2):
        self.filenames = train_lib[:,0]
        self.labels = train_lib[:,1]
        self.transform = transform
        self.transform2 = transform2
        #self.new_feature = 

    def __len__(self):
        return len(self.filenames)

    def __getitem__(self, idx):
        img = Image.open("../input/imet-2019-fgvc6/train/"+format(self.filenames[idx])+'.png')  # PIL image
        #image= image.filter(ImageFilter.EDGE_ENHANCE)
        #image2 = image.filter(ImageFilter.FIND_EDGES)
        image = self.transform(img)
        image2 = self.transform2(img)
        #return image, self.labels[idx]
        return image,image2, self.labels[idx]
In [19]:
class testDataset(Dataset):
    def __init__(self, test_lib, transform,transform2):
        test_lib = np.array(test_lib)
        self.filenames = test_lib[:,0]
        #self.labels = test_lib[:,1]
        self.transform = transform
        self.transform2 = transform2

    def __len__(self):
        return len(self.filenames)

    def __getitem__(self, idx):
        img = Image.open("../input/imet-2019-fgvc6/test/"+format(self.filenames[idx])+'.png')  # PIL image
        #image= image.filter(ImageFilter.EDGE_ENHANCE)
        #image2 = image.filter(ImageFilter.FIND_EDGES)
        image = self.transform(img)
        image2 = self.transform2(img)
        #return image,self.filenames[idx]
        return image,image2,self.filenames[idx]
In [20]:
trainA_dataloader = DataLoader(trainDataset(datafilter, data_transforms,data_transforms2),batch_size=32, shuffle=True,num_workers=2, pin_memory=True)
#trainC_dataloader = DataLoader(trainDataset(train_c, data_transforms,data_transforms2),batch_size=32, shuffle=True,num_workers=2, pin_memory=True)
#trainT_dataloader = DataLoader(trainDataset(train_t, data_transforms,data_transforms2),batch_size=32, shuffle=True,num_workers=2, pin_memory=True)
In [21]:
score_dataloader = DataLoader(trainDataset(datafilter[:1000], data_transforms,data_transforms2),batch_size=32, shuffle=False,num_workers=2, pin_memory=True)
In [22]:
test_dataloader = DataLoader(testDataset(test, data_transforms,data_transforms2),batch_size=32,shuffle=False,num_workers=2, pin_memory=True)
In [23]:
plt.figure(figsize=(10,10))
plt.subplot(2,2,1)
plt.imshow(transforms.ToPILImage()(trainA_dataloader.dataset[15][0]).convert('RGB'))
plt.subplot(2,2,2)
plt.imshow(transforms.ToPILImage()(trainA_dataloader.dataset[15][1]).convert('RGB'))
plt.subplot(2,2,3)
plt.imshow(transforms.ToPILImage()(trainA_dataloader.dataset[29][0]).convert('RGB'))
plt.subplot(2,2,4)
plt.imshow(transforms.ToPILImage()(trainA_dataloader.dataset[29][1]).convert('RGB'))
Out[23]:
<matplotlib.image.AxesImage at 0x7fa8e8f1a588>
In [24]:
#(np.array(trainC_dataloader.dataset[4][0]).shape)
In [25]:
# https://github.com/Cadene/pretrained-models.pytorch/blob/master/pretrainedmodels/models/senet.py
"""
ResNet code gently borrowed from
https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py
"""
from __future__ import print_function, division, absolute_import
from collections import OrderedDict
import math

import torch.nn as nn
from torch.utils import model_zoo

__all__ = ['SENet', 'senet154', 'se_resnet50', 'se_resnet101', 'se_resnet152',
           'se_resnext50_32x4d', 'se_resnext101_32x4d']

pretrained_settings = {
    'senet154': {
        'imagenet': {
            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/senet154-c7b49a05.pth',
            'input_space': 'RGB',
            'input_size': [3, 224, 224],
            'input_range': [0, 1],
            'mean': [0.485, 0.456, 0.406],
            'std': [0.229, 0.224, 0.225],
            'num_classes': 1000
        }
    },
    'se_resnet50': {
        'imagenet': {
            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet50-ce0d4300.pth',
            'input_space': 'RGB',
            'input_size': [3, 224, 224],
            'input_range': [0, 1],
            'mean': [0.485, 0.456, 0.406],
            'std': [0.229, 0.224, 0.225],
            'num_classes': 1000
        }
    },
    'se_resnet101': {
        'imagenet': {
            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet101-7e38fcc6.pth',
            'input_space': 'RGB',
            'input_size': [3, 224, 224],
            'input_range': [0, 1],
            'mean': [0.485, 0.456, 0.406],
            'std': [0.229, 0.224, 0.225],
            'num_classes': 1000
        }
    },
    'se_resnet152': {
        'imagenet': {
            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet152-d17c99b7.pth',
            'input_space': 'RGB',
            'input_size': [3, 224, 224],
            'input_range': [0, 1],
            'mean': [0.485, 0.456, 0.406],
            'std': [0.229, 0.224, 0.225],
            'num_classes': 1000
        }
    },
    'se_resnext50_32x4d': {
        'imagenet': {
            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext50_32x4d-a260b3a4.pth',
            'input_space': 'RGB',
            'input_size': [3, 224, 224],
            'input_range': [0, 1],
            'mean': [0.485, 0.456, 0.406],
            'std': [0.229, 0.224, 0.225],
            'num_classes': 1000
        }
    },
    'se_resnext101_32x4d': {
        'imagenet': {
            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext101_32x4d-3b2fe3d8.pth',
            'input_space': 'RGB',
            'input_size': [3, 224, 224],
            'input_range': [0, 1],
            'mean': [0.485, 0.456, 0.406],
            'std': [0.229, 0.224, 0.225],
            'num_classes': 1000
        }
    },
}


class SEModule(nn.Module):

    def __init__(self, channels, reduction):
        super(SEModule, self).__init__()
        self.avg_pool = nn.AdaptiveAvgPool2d(1)
        self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,
                             padding=0)
        self.relu = nn.ReLU(inplace=True)
        self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,
                             padding=0)
        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        module_input = x
        x = self.avg_pool(x)
        x = self.fc1(x)
        x = self.relu(x)
        x = self.fc2(x)
        x = self.sigmoid(x)
        return module_input * x


class Bottleneck(nn.Module):
    """
    Base class for bottlenecks that implements `forward()` method.
    """
    def forward(self, x):
        residual = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu(out)

        out = self.conv3(out)
        out = self.bn3(out)

        if self.downsample is not None:
            residual = self.downsample(x)

        out = self.se_module(out) + residual
        out = self.relu(out)

        return out


class SEBottleneck(Bottleneck):
    """
    Bottleneck for SENet154.
    """
    expansion = 4

    def __init__(self, inplanes, planes, groups, reduction, stride=1,
                 downsample=None):
        super(SEBottleneck, self).__init__()
        self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)
        self.bn1 = nn.BatchNorm2d(planes * 2)
        self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,
                               stride=stride, padding=1, groups=groups,
                               bias=False)
        self.bn2 = nn.BatchNorm2d(planes * 4)
        self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,
                               bias=False)
        self.bn3 = nn.BatchNorm2d(planes * 4)
        self.relu = nn.ReLU(inplace=True)
        self.se_module = SEModule(planes * 4, reduction=reduction)
        self.downsample = downsample
        self.stride = stride


class SEResNetBottleneck(Bottleneck):
    """
    ResNet bottleneck with a Squeeze-and-Excitation module. It follows Caffe
    implementation and uses `stride=stride` in `conv1` and not in `conv2`
    (the latter is used in the torchvision implementation of ResNet).
    """
    expansion = 4

    def __init__(self, inplanes, planes, groups, reduction, stride=1,
                 downsample=None):
        super(SEResNetBottleneck, self).__init__()
        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False,
                               stride=stride)
        self.bn1 = nn.BatchNorm2d(planes)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1,
                               groups=groups, bias=False)
        self.bn2 = nn.BatchNorm2d(planes)
        self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
        self.bn3 = nn.BatchNorm2d(planes * 4)
        self.relu = nn.ReLU(inplace=True)
        self.se_module = SEModule(planes * 4, reduction=reduction)
        self.downsample = downsample
        self.stride = stride


class SEResNeXtBottleneck(Bottleneck):
    """
    ResNeXt bottleneck type C with a Squeeze-and-Excitation module.
    """
    expansion = 4

    def __init__(self, inplanes, planes, groups, reduction, stride=1,
                 downsample=None, base_width=4):
        super(SEResNeXtBottleneck, self).__init__()
        width = math.floor(planes * (base_width / 64)) * groups
        self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False,
                               stride=1)
        self.bn1 = nn.BatchNorm2d(width)
        self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride,
                               padding=1, groups=groups, bias=False)
        self.bn2 = nn.BatchNorm2d(width)
        self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)
        self.bn3 = nn.BatchNorm2d(planes * 4)
        self.relu = nn.ReLU(inplace=True)
        self.se_module = SEModule(planes * 4, reduction=reduction)
        self.downsample = downsample
        self.stride = stride


class SENet(nn.Module):

    def __init__(self, block, layers, groups, reduction, dropout_p=0.2,
                 inplanes=128, input_3x3=True, downsample_kernel_size=3,
                 downsample_padding=1, num_classes=1000):
        """
        Parameters
        ----------
        block (nn.Module): Bottleneck class.
            - For SENet154: SEBottleneck
            - For SE-ResNet models: SEResNetBottleneck
            - For SE-ResNeXt models:  SEResNeXtBottleneck
        layers (list of ints): Number of residual blocks for 4 layers of the
            network (layer1...layer4).
        groups (int): Number of groups for the 3x3 convolution in each
            bottleneck block.
            - For SENet154: 64
            - For SE-ResNet models: 1
            - For SE-ResNeXt models:  32
        reduction (int): Reduction ratio for Squeeze-and-Excitation modules.
            - For all models: 16
        dropout_p (float or None): Drop probability for the Dropout layer.
            If `None` the Dropout layer is not used.
            - For SENet154: 0.2
            - For SE-ResNet models: None
            - For SE-ResNeXt models: None
        inplanes (int):  Number of input channels for layer1.
            - For SENet154: 128
            - For SE-ResNet models: 64
            - For SE-ResNeXt models: 64
        input_3x3 (bool): If `True`, use three 3x3 convolutions instead of
            a single 7x7 convolution in layer0.
            - For SENet154: True
            - For SE-ResNet models: False
            - For SE-ResNeXt models: False
        downsample_kernel_size (int): Kernel size for downsampling convolutions
            in layer2, layer3 and layer4.
            - For SENet154: 3
            - For SE-ResNet models: 1
            - For SE-ResNeXt models: 1
        downsample_padding (int): Padding for downsampling convolutions in
            layer2, layer3 and layer4.
            - For SENet154: 1
            - For SE-ResNet models: 0
            - For SE-ResNeXt models: 0
        num_classes (int): Number of outputs in `last_linear` layer.
            - For all models: 1000
        """
        super(SENet, self).__init__()
        self.inplanes = inplanes
        if input_3x3:
            layer0_modules = [
                ('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1,
                                    bias=False)),
                ('bn1', nn.BatchNorm2d(64)),
                ('relu1', nn.ReLU(inplace=True)),
                ('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,
                                    bias=False)),
                ('bn2', nn.BatchNorm2d(64)),
                ('relu2', nn.ReLU(inplace=True)),
                ('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,
                                    bias=False)),
                ('bn3', nn.BatchNorm2d(inplanes)),
                ('relu3', nn.ReLU(inplace=True)),
            ]
        else:
            layer0_modules = [
                ('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,
                                    padding=3, bias=False)),
                ('bn1', nn.BatchNorm2d(inplanes)),
                ('relu1', nn.ReLU(inplace=True)),
            ]
        # To preserve compatibility with Caffe weights `ceil_mode=True`
        # is used instead of `padding=1`.
        layer0_modules.append(('pool', nn.MaxPool2d(3, stride=2,
                                                    ceil_mode=True)))
        self.layer0 = nn.Sequential(OrderedDict(layer0_modules))
        self.layer1 = self._make_layer(
            block,
            planes=64,
            blocks=layers[0],
            groups=groups,
            reduction=reduction,
            downsample_kernel_size=1,
            downsample_padding=0
        )
        self.layer2 = self._make_layer(
            block,
            planes=128,
            blocks=layers[1],
            stride=2,
            groups=groups,
            reduction=reduction,
            downsample_kernel_size=downsample_kernel_size,
            downsample_padding=downsample_padding
        )
        self.layer3 = self._make_layer(
            block,
            planes=256,
            blocks=layers[2],
            stride=2,
            groups=groups,
            reduction=reduction,
            downsample_kernel_size=downsample_kernel_size,
            downsample_padding=downsample_padding
        )
        self.layer4 = self._make_layer(
            block,
            planes=512,
            blocks=layers[3],
            stride=2,
            groups=groups,
            reduction=reduction,
            downsample_kernel_size=downsample_kernel_size,
            downsample_padding=downsample_padding
        )
        self.avg_pool = nn.AvgPool2d(7, stride=1)
        self.dropout = nn.Dropout(dropout_p) if dropout_p is not None else None
        self.last_linear = nn.Linear(512 * block.expansion, num_classes)

    def _make_layer(self, block, planes, blocks, groups, reduction, stride=1,
                    downsample_kernel_size=1, downsample_padding=0):
        downsample = None
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                nn.Conv2d(self.inplanes, planes * block.expansion,
                          kernel_size=downsample_kernel_size, stride=stride,
                          padding=downsample_padding, bias=False),
                nn.BatchNorm2d(planes * block.expansion),
            )

        layers = []
        layers.append(block(self.inplanes, planes, groups, reduction, stride,
                            downsample))
        self.inplanes = planes * block.expansion
        for i in range(1, blocks):
            layers.append(block(self.inplanes, planes, groups, reduction))

        return nn.Sequential(*layers)

    def features(self, x):
        x = self.layer0(x)
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)
        return x

    def logits(self, x):
        x = self.avg_pool(x)
        if self.dropout is not None:
            x = self.dropout(x)
        x = x.view(x.size(0), -1)
        x = self.last_linear(x)
        return x

    def forward(self, x):
        x = self.features(x)
        x = self.logits(x)
        return x


def initialize_pretrained_model(model, num_classes, settings):
    assert num_classes == settings['num_classes'], \
        'num_classes should be {}, but is {}'.format(
            settings['num_classes'], num_classes)
    model.load_state_dict(model_zoo.load_url(settings['url']))
    model.input_space = settings['input_space']
    model.input_size = settings['input_size']
    model.input_range = settings['input_range']
    model.mean = settings['mean']
    model.std = settings['std']


def senet154(num_classes=1000, pretrained='imagenet'):
    model = SENet(SEBottleneck, [3, 8, 36, 3], groups=64, reduction=16,
                  dropout_p=0.2, num_classes=num_classes)
    if pretrained is not None:
        settings = pretrained_settings['senet154'][pretrained]
        initialize_pretrained_model(model, num_classes, settings)
    return model


def se_resnet50(num_classes=1000, pretrained='imagenet'):
    model = SENet(SEResNetBottleneck, [3, 4, 6, 3], groups=1, reduction=16,
                  dropout_p=None, inplanes=64, input_3x3=False,
                  downsample_kernel_size=1, downsample_padding=0,
                  num_classes=num_classes)
    if pretrained is not None:
        settings = pretrained_settings['se_resnet50'][pretrained]
        initialize_pretrained_model(model, num_classes, settings)
    return model


def se_resnet101(num_classes=1000, pretrained='imagenet'):
    model = SENet(SEResNetBottleneck, [3, 4, 23, 3], groups=1, reduction=16,
                  dropout_p=None, inplanes=64, input_3x3=False,
                  downsample_kernel_size=1, downsample_padding=0,
                  num_classes=num_classes)
    if pretrained is not None:
        settings = pretrained_settings['se_resnet101'][pretrained]
        initialize_pretrained_model(model, num_classes, settings)
    return model


def se_resnet152(num_classes=1000, pretrained='imagenet'):
    model = SENet(SEResNetBottleneck, [3, 8, 36, 3], groups=1, reduction=16,
                  dropout_p=None, inplanes=64, input_3x3=False,
                  downsample_kernel_size=1, downsample_padding=0,
                  num_classes=num_classes)
    if pretrained is not None:
        settings = pretrained_settings['se_resnet152'][pretrained]
        initialize_pretrained_model(model, num_classes, settings)
    return model


def se_resnext50_32x4d(num_classes=1000, pretrained='imagenet'):
    model = SENet(SEResNeXtBottleneck, [3, 4, 6, 3], groups=32, reduction=16,
                  dropout_p=None, inplanes=64, input_3x3=False,
                  downsample_kernel_size=1, downsample_padding=0,
                  num_classes=num_classes)
    if pretrained is not None:
        settings = pretrained_settings['se_resnext50_32x4d'][pretrained]
        initialize_pretrained_model(model, num_classes, settings)
    return model


def se_resnext101_32x4d(num_classes=1000, pretrained='imagenet'):
    model = SENet(SEResNeXtBottleneck, [3, 4, 23, 3], groups=32, reduction=16,
                  dropout_p=None, inplanes=64, input_3x3=False,
                  downsample_kernel_size=1, downsample_padding=0,
                  num_classes=num_classes)
    if pretrained is not None:
        settings = pretrained_settings['se_resnext101_32x4d'][pretrained]
        initialize_pretrained_model(model, num_classes, settings)
    return model
In [26]:
from shutil import copyfile
from os.path import isfile, join, abspath, exists, isdir, expanduser
from os import listdir, makedirs, getcwd, remove

cache_dir = expanduser(join('~', '.torch'))
if not exists(cache_dir):
    makedirs(cache_dir)
models_dir = join(cache_dir, 'models/')
if not exists(models_dir):
    makedirs(models_dir)
    
copyfile("../input/pretrained-pytorch/densenet201-c1103571.pth", models_dir+"densenet201-c1103571.pth")
copyfile("../input/pretrained-pytorch/resnet50-19c8e357.pth", models_dir+"resnet50-19c8e357.pth")
copyfile("../input/pretrained-se-resnet/se_resnext101_32x4d-3b2fe3d8.pth", models_dir+"se_resnext101_32x4d-3b2fe3d8.pth")
Out[26]:
'/tmp/.torch/models/se_resnext101_32x4d-3b2fe3d8.pth'
In [27]:
!ls ~/.torch/models 
densenet201-c1103571.pth  se_resnext101_32x4d-3b2fe3d8.pth
resnet50-19c8e357.pth
In [28]:
class Ensemble(nn.Module):
    def __init__(self, modelA, modelB,input_length,output_length):
        super(Ensemble, self).__init__()
        self.modelA = modelA
        self.modelB = modelB
        self.classifier = nn.Linear(input_length, output_length)
        
    def forward(self, xin,xin2):
        x1 = self.modelA(xin)
        x2 = self.modelB(xin2)
        x = torch.cat((x1, x2), dim=1)
        x = self.classifier(F.relu(x))
        return x

    
# efficientNet = EfficientNet.from_name('efficientnet-b7') 
# efficientNet._fc = nn.Linear(in_features=1280, out_features=lable_length)


seresnext_model = se_resnext101_32x4d(pretrained='imagenet')
seresnext_model.last_linear = nn.Linear(in_features=2048, out_features=lable_length, bias=True)

#densenet_model = models.densenet201(pretrained=True)
#densenet_model.load_state_dict(torch.load( models_dir+"densenet201-c1103571.pth"))
#densenet_model.classifier= nn.Linear(in_features=1920,out_features=lable_length)

#efficientNet = EfficientNet(width_coeff=2.0, depth_coeff=3.1,drop_connect_rate=0.5,num_classes = lable_length)

resnet_model = models.resnet50(pretrained=True)
#resnet_model.load_state_dict(torch.load("../models/resnet50.pth"))
resnet_model.fc= nn.Linear(in_features=2048, out_features=lable_length)

#model = Ensemble(seresnext_model, resnet_model,lable_length*2,lable_length)
#model.to(device)

model = seresnext_model
model.to(device)
Out[28]:
SENet(
  (layer0): Sequential(
    (conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
    (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (relu1): ReLU(inplace)
    (pool): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=True)
  )
  (layer1): Sequential(
    (0): SEResNeXtBottleneck(
      (conv1): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(128, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(256, 16, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(16, 256, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
      (downsample): Sequential(
        (0): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (1): SEResNeXtBottleneck(
      (conv1): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(128, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(256, 16, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(16, 256, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (2): SEResNeXtBottleneck(
      (conv1): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(128, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(256, 16, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(16, 256, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
  )
  (layer2): Sequential(
    (0): SEResNeXtBottleneck(
      (conv1): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(256, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(512, 32, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(32, 512, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
      (downsample): Sequential(
        (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
        (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (1): SEResNeXtBottleneck(
      (conv1): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(256, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(512, 32, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(32, 512, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (2): SEResNeXtBottleneck(
      (conv1): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(256, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(512, 32, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(32, 512, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (3): SEResNeXtBottleneck(
      (conv1): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(256, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(512, 32, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(32, 512, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
  )
  (layer3): Sequential(
    (0): SEResNeXtBottleneck(
      (conv1): Conv2d(512, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(1024, 64, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(64, 1024, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
      (downsample): Sequential(
        (0): Conv2d(512, 1024, kernel_size=(1, 1), stride=(2, 2), bias=False)
        (1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (1): SEResNeXtBottleneck(
      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(1024, 64, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(64, 1024, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (2): SEResNeXtBottleneck(
      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(1024, 64, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(64, 1024, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (3): SEResNeXtBottleneck(
      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(1024, 64, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(64, 1024, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (4): SEResNeXtBottleneck(
      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(1024, 64, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(64, 1024, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (5): SEResNeXtBottleneck(
      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(1024, 64, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(64, 1024, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (6): SEResNeXtBottleneck(
      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(1024, 64, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(64, 1024, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (7): SEResNeXtBottleneck(
      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(1024, 64, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(64, 1024, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (8): SEResNeXtBottleneck(
      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(1024, 64, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(64, 1024, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (9): SEResNeXtBottleneck(
      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(1024, 64, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(64, 1024, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (10): SEResNeXtBottleneck(
      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(1024, 64, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(64, 1024, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (11): SEResNeXtBottleneck(
      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(1024, 64, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(64, 1024, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (12): SEResNeXtBottleneck(
      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(1024, 64, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(64, 1024, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (13): SEResNeXtBottleneck(
      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(1024, 64, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(64, 1024, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (14): SEResNeXtBottleneck(
      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(1024, 64, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(64, 1024, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (15): SEResNeXtBottleneck(
      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(1024, 64, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(64, 1024, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (16): SEResNeXtBottleneck(
      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(1024, 64, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(64, 1024, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (17): SEResNeXtBottleneck(
      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(1024, 64, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(64, 1024, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (18): SEResNeXtBottleneck(
      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(1024, 64, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(64, 1024, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (19): SEResNeXtBottleneck(
      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(1024, 64, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(64, 1024, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (20): SEResNeXtBottleneck(
      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(1024, 64, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(64, 1024, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (21): SEResNeXtBottleneck(
      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(1024, 64, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(64, 1024, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (22): SEResNeXtBottleneck(
      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(1024, 64, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(64, 1024, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
  )
  (layer4): Sequential(
    (0): SEResNeXtBottleneck(
      (conv1): Conv2d(1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(1024, 1024, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(1024, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(2048, 128, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(128, 2048, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
      (downsample): Sequential(
        (0): Conv2d(1024, 2048, kernel_size=(1, 1), stride=(2, 2), bias=False)
        (1): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (1): SEResNeXtBottleneck(
      (conv1): Conv2d(2048, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(1024, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(2048, 128, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(128, 2048, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
    (2): SEResNeXtBottleneck(
      (conv1): Conv2d(2048, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
      (bn2): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(1024, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (se_module): SEModule(
        (avg_pool): AdaptiveAvgPool2d(output_size=1)
        (fc1): Conv2d(2048, 128, kernel_size=(1, 1), stride=(1, 1))
        (relu): ReLU(inplace)
        (fc2): Conv2d(128, 2048, kernel_size=(1, 1), stride=(1, 1))
        (sigmoid): Sigmoid()
      )
    )
  )
  (avg_pool): AvgPool2d(kernel_size=7, stride=1, padding=0)
  (last_linear): Linear(in_features=2048, out_features=1103, bias=True)
)
In [29]:
def train(epoch,fnum,dloader):
    model.train()
    for step, (x,x2,y) in enumerate(dloader):
        data = Variable(x).cuda()   # batch x
        #data2 = Variable(x2).cuda()
        target = Variable(y).cuda()   # batch y
        #print(data.size())
        #print(data2.size())
        #print(target.size())
        #output = model(data,data2)
        output = model(data)

        loss = loss_func(output, target.float())   # cross entropy loss
        optimizer.zero_grad()           # clear gradients for this training step
        loss.backward()                 # backpropagation, compute gradients
        optimizer.step()                # apply gradients
        if step==0:
            start = time.time()
            #break
            ti = 0
        elif step==100:
            ti = time.time()-start #total time = ti*(length/100)
            #print(ti)
            ti = ti*(len(dloader)/100)
        if step % 100 == 0:
            second = ti*(((len(dloader)-step)/len(dloader)))#*(5-epoch)*(4-fnum)
            print('Train Fold:{}/4  Ep: {}/3 [{}/{} ({:.0f}%)]\t Loss: {:.6f}\t Remain : {} '.
                     format(fnum+1,
                            epoch+1, 
                            step * len(data), 
                            len(dloader.dataset),
                            100.*step/len(dloader), 
                            loss.data.item(),
                            datetime.timedelta(seconds = int(second))))
        data.cpu()
        #data2.cpu()
        target.cpu()
        torch.cuda.empty_cache()
    print("Finish")
In [30]:
def val(dloader):
    model.eval()
    los = []
    for step, (x,x2, y) in enumerate(dloader):
        data = Variable(x).cuda()
        #data2 = Variable(x2).cuda()
        target = Variable(y).cuda()
        
        #output = model(data,data2)
        output = model(data)
        
        loss = loss_func(output, target.float())
        los.append(loss.item())
        
        
        if step %100 == 0:
            print('[{}/{} ({:.1f}%)]'.format(step * len(data), 
                                        len(dloader.dataset),
                                        100.*step/len(dloader)))
        data.cpu()
        #data2.cpu()
        target.cpu()
        torch.cuda.empty_cache()
    los = np.array(los)
    avg_val_loss = los.sum()/len(los)
    print(f"Avg val loss: {avg_val_loss:.8f}")
    #return ans,out
    
In [31]:
class FocalLoss(nn.Module):
    def __init__(self, gamma):
        super().__init__()
        self.gamma = gamma
        
    def forward(self, input, target):
        # Inspired by the implementation of binary_cross_entropy_with_logits
        if not (target.size() == input.size()):
            raise ValueError("Target size ({}) must be the same as input size ({})".format(target.size(), input.size()))

        max_val = (-input).clamp(min=0)
        loss = input - input * target + max_val + ((-max_val).exp() + (-input - max_val).exp()).log()

        # This formula gives us the log sigmoid of 1-p if y is 0 and of p if y is 1
        invprobs = F.logsigmoid(-input * (target * 2 - 1))
        loss = (invprobs * self.gamma).exp() * loss
        
        return loss.mean()
In [32]:
fold = KFold(n_splits = 4, random_state = 10)
for fold_num, (trn_idx, val_idx) in enumerate(fold.split(datafilter)):
    Ftrain_dataloader = datafilter[trn_idx, :]
    Fval_dataloader = datafilter[val_idx, :]

    val_dataloader = DataLoader(trainDataset(Fval_dataloader, data_transforms,data_transforms2),batch_size=32, shuffle=False,num_workers=2, pin_memory=True)
    train_dataloader = DataLoader(trainDataset(Ftrain_dataloader, data_transforms,data_transforms2),batch_size=32, shuffle=True,num_workers=2, pin_memory=True)
    for epoch in range(3):
        ###########################################
        if epoch==0:
            optimizer = torch.optim.Adam(model.parameters(), lr=0.0001/(2**epoch))
        else:
            optimizer = torch.optim.SGD(model.parameters(), lr=0.0001,momentum=0.9, weight_decay=1e-4)
        #optimizer = torch.optim.Adam(model.parameters(), lr=0.00002/(2**epoch))
        #optimizer = torch.optim.SGD(net.parameters(), lr=args.lr, momentum=0.9, weight_decay=1e-4)
        loss_func = FocalLoss(0.4)
        #loss_func = torch.nn.MSELoss()
        ###########################################
        train(epoch,fold_num,train_dataloader) 
        val(val_dataloader)
Train Fold:1/4  Ep: 1/3 [0/81927 (0%)]	 Loss: 0.537626	 Remain : 0:00:00 
Train Fold:1/4  Ep: 1/3 [3200/81927 (4%)]	 Loss: 0.011705	 Remain : 0:33:39 
Train Fold:1/4  Ep: 1/3 [6400/81927 (8%)]	 Loss: 0.009185	 Remain : 0:32:17 
Train Fold:1/4  Ep: 1/3 [9600/81927 (12%)]	 Loss: 0.010711	 Remain : 0:30:55 
Train Fold:1/4  Ep: 1/3 [12800/81927 (16%)]	 Loss: 0.008606	 Remain : 0:29:33 
Train Fold:1/4  Ep: 1/3 [16000/81927 (20%)]	 Loss: 0.008366	 Remain : 0:28:11 
Train Fold:1/4  Ep: 1/3 [19200/81927 (23%)]	 Loss: 0.008048	 Remain : 0:26:49 
Train Fold:1/4  Ep: 1/3 [22400/81927 (27%)]	 Loss: 0.009078	 Remain : 0:25:27 
Train Fold:1/4  Ep: 1/3 [25600/81927 (31%)]	 Loss: 0.008839	 Remain : 0:24:05 
Train Fold:1/4  Ep: 1/3 [28800/81927 (35%)]	 Loss: 0.007164	 Remain : 0:22:43 
Train Fold:1/4  Ep: 1/3 [32000/81927 (39%)]	 Loss: 0.007610	 Remain : 0:21:21 
Train Fold:1/4  Ep: 1/3 [35200/81927 (43%)]	 Loss: 0.008272	 Remain : 0:19:59 
Train Fold:1/4  Ep: 1/3 [38400/81927 (47%)]	 Loss: 0.009426	 Remain : 0:18:37 
Train Fold:1/4  Ep: 1/3 [41600/81927 (51%)]	 Loss: 0.007567	 Remain : 0:17:15 
Train Fold:1/4  Ep: 1/3 [44800/81927 (55%)]	 Loss: 0.008095	 Remain : 0:15:52 
Train Fold:1/4  Ep: 1/3 [48000/81927 (59%)]	 Loss: 0.007952	 Remain : 0:14:30 
Train Fold:1/4  Ep: 1/3 [51200/81927 (62%)]	 Loss: 0.007757	 Remain : 0:13:08 
Train Fold:1/4  Ep: 1/3 [54400/81927 (66%)]	 Loss: 0.008596	 Remain : 0:11:46 
Train Fold:1/4  Ep: 1/3 [57600/81927 (70%)]	 Loss: 0.007792	 Remain : 0:10:24 
Train Fold:1/4  Ep: 1/3 [60800/81927 (74%)]	 Loss: 0.007988	 Remain : 0:09:02 
Train Fold:1/4  Ep: 1/3 [64000/81927 (78%)]	 Loss: 0.007846	 Remain : 0:07:40 
Train Fold:1/4  Ep: 1/3 [67200/81927 (82%)]	 Loss: 0.007322	 Remain : 0:06:18 
Train Fold:1/4  Ep: 1/3 [70400/81927 (86%)]	 Loss: 0.008704	 Remain : 0:04:56 
Train Fold:1/4  Ep: 1/3 [73600/81927 (90%)]	 Loss: 0.006315	 Remain : 0:03:34 
Train Fold:1/4  Ep: 1/3 [76800/81927 (94%)]	 Loss: 0.007462	 Remain : 0:02:12 
Train Fold:1/4  Ep: 1/3 [80000/81927 (98%)]	 Loss: 0.007671	 Remain : 0:00:50 
Finish
[0/27310 (0.0%)]
[3200/27310 (11.7%)]
[6400/27310 (23.4%)]
[9600/27310 (35.1%)]
[12800/27310 (46.8%)]
[16000/27310 (58.5%)]
[19200/27310 (70.3%)]
[22400/27310 (82.0%)]
[25600/27310 (93.7%)]
Avg val loss: 0.00687568
Train Fold:1/4  Ep: 2/3 [0/81927 (0%)]	 Loss: 0.007003	 Remain : 0:00:00 
Train Fold:1/4  Ep: 2/3 [3200/81927 (4%)]	 Loss: 0.007793	 Remain : 0:33:53 
Train Fold:1/4  Ep: 2/3 [6400/81927 (8%)]	 Loss: 0.007627	 Remain : 0:32:31 
Train Fold:1/4  Ep: 2/3 [9600/81927 (12%)]	 Loss: 0.007228	 Remain : 0:31:08 
Train Fold:1/4  Ep: 2/3 [12800/81927 (16%)]	 Loss: 0.006818	 Remain : 0:29:45 
Train Fold:1/4  Ep: 2/3 [16000/81927 (20%)]	 Loss: 0.006233	 Remain : 0:28:23 
Train Fold:1/4  Ep: 2/3 [19200/81927 (23%)]	 Loss: 0.005438	 Remain : 0:27:00 
Train Fold:1/4  Ep: 2/3 [22400/81927 (27%)]	 Loss: 0.005480	 Remain : 0:25:37 
Train Fold:1/4  Ep: 2/3 [25600/81927 (31%)]	 Loss: 0.006868	 Remain : 0:24:15 
Train Fold:1/4  Ep: 2/3 [28800/81927 (35%)]	 Loss: 0.007285	 Remain : 0:22:52 
Train Fold:1/4  Ep: 2/3 [32000/81927 (39%)]	 Loss: 0.006798	 Remain : 0:21:30 
Train Fold:1/4  Ep: 2/3 [35200/81927 (43%)]	 Loss: 0.005700	 Remain : 0:20:07 
Train Fold:1/4  Ep: 2/3 [38400/81927 (47%)]	 Loss: 0.006014	 Remain : 0:18:44 
Train Fold:1/4  Ep: 2/3 [41600/81927 (51%)]	 Loss: 0.005580	 Remain : 0:17:22 
Train Fold:1/4  Ep: 2/3 [44800/81927 (55%)]	 Loss: 0.004795	 Remain : 0:15:59 
Train Fold:1/4  Ep: 2/3 [48000/81927 (59%)]	 Loss: 0.007108	 Remain : 0:14:36 
Train Fold:1/4  Ep: 2/3 [51200/81927 (62%)]	 Loss: 0.005633	 Remain : 0:13:14 
Train Fold:1/4  Ep: 2/3 [54400/81927 (66%)]	 Loss: 0.005324	 Remain : 0:11:51 
Train Fold:1/4  Ep: 2/3 [57600/81927 (70%)]	 Loss: 0.007533	 Remain : 0:10:28 
Train Fold:1/4  Ep: 2/3 [60800/81927 (74%)]	 Loss: 0.006739	 Remain : 0:09:06 
Train Fold:1/4  Ep: 2/3 [64000/81927 (78%)]	 Loss: 0.006118	 Remain : 0:07:43 
Train Fold:1/4  Ep: 2/3 [67200/81927 (82%)]	 Loss: 0.006273	 Remain : 0:06:20 
Train Fold:1/4  Ep: 2/3 [70400/81927 (86%)]	 Loss: 0.005824	 Remain : 0:04:58 
Train Fold:1/4  Ep: 2/3 [73600/81927 (90%)]	 Loss: 0.005992	 Remain : 0:03:35 
Train Fold:1/4  Ep: 2/3 [76800/81927 (94%)]	 Loss: 0.006129	 Remain : 0:02:13 
Train Fold:1/4  Ep: 2/3 [80000/81927 (98%)]	 Loss: 0.006127	 Remain : 0:00:50 
Finish
[0/27310 (0.0%)]
[3200/27310 (11.7%)]
[6400/27310 (23.4%)]
[9600/27310 (35.1%)]
[12800/27310 (46.8%)]
[16000/27310 (58.5%)]
[19200/27310 (70.3%)]
[22400/27310 (82.0%)]
[25600/27310 (93.7%)]
Avg val loss: 0.00684409
Train Fold:1/4  Ep: 3/3 [0/81927 (0%)]	 Loss: 0.007810	 Remain : 0:00:00 
Train Fold:1/4  Ep: 3/3 [3200/81927 (4%)]	 Loss: 0.005828	 Remain : 0:32:18 
Train Fold:1/4  Ep: 3/3 [6400/81927 (8%)]	 Loss: 0.006158	 Remain : 0:30:59 
Train Fold:1/4  Ep: 3/3 [9600/81927 (12%)]	 Loss: 0.006283	 Remain : 0:29:40 
Train Fold:1/4  Ep: 3/3 [12800/81927 (16%)]	 Loss: 0.006567	 Remain : 0:28:21 
Train Fold:1/4  Ep: 3/3 [16000/81927 (20%)]	 Loss: 0.005497	 Remain : 0:27:03 
Train Fold:1/4  Ep: 3/3 [19200/81927 (23%)]	 Loss: 0.006548	 Remain : 0:25:44 
Train Fold:1/4  Ep: 3/3 [22400/81927 (27%)]	 Loss: 0.006745	 Remain : 0:24:25 
Train Fold:1/4  Ep: 3/3 [25600/81927 (31%)]	 Loss: 0.005836	 Remain : 0:23:06 
Train Fold:1/4  Ep: 3/3 [28800/81927 (35%)]	 Loss: 0.007159	 Remain : 0:21:48 
Train Fold:1/4  Ep: 3/3 [32000/81927 (39%)]	 Loss: 0.006357	 Remain : 0:20:29 
Train Fold:1/4  Ep: 3/3 [35200/81927 (43%)]	 Loss: 0.006810	 Remain : 0:19:10 
Train Fold:1/4  Ep: 3/3 [38400/81927 (47%)]	 Loss: 0.005392	 Remain : 0:17:51 
Train Fold:1/4  Ep: 3/3 [41600/81927 (51%)]	 Loss: 0.006752	 Remain : 0:16:33 
Train Fold:1/4  Ep: 3/3 [44800/81927 (55%)]	 Loss: 0.006994	 Remain : 0:15:14 
Train Fold:1/4  Ep: 3/3 [48000/81927 (59%)]	 Loss: 0.007724	 Remain : 0:13:55 
Train Fold:1/4  Ep: 3/3 [51200/81927 (62%)]	 Loss: 0.006599	 Remain : 0:12:36 
Train Fold:1/4  Ep: 3/3 [54400/81927 (66%)]	 Loss: 0.005863	 Remain : 0:11:18 
Train Fold:1/4  Ep: 3/3 [57600/81927 (70%)]	 Loss: 0.006200	 Remain : 0:09:59 
Train Fold:1/4  Ep: 3/3 [60800/81927 (74%)]	 Loss: 0.006622	 Remain : 0:08:40 
Train Fold:1/4  Ep: 3/3 [64000/81927 (78%)]	 Loss: 0.006256	 Remain : 0:07:21 
Train Fold:1/4  Ep: 3/3 [67200/81927 (82%)]	 Loss: 0.006534	 Remain : 0:06:03 
Train Fold:1/4  Ep: 3/3 [70400/81927 (86%)]	 Loss: 0.006046	 Remain : 0:04:44 
Train Fold:1/4  Ep: 3/3 [73600/81927 (90%)]	 Loss: 0.007525	 Remain : 0:03:25 
Train Fold:1/4  Ep: 3/3 [76800/81927 (94%)]	 Loss: 0.007216	 Remain : 0:02:06 
Train Fold:1/4  Ep: 3/3 [80000/81927 (98%)]	 Loss: 0.005830	 Remain : 0:00:48 
Finish
[0/27310 (0.0%)]
[3200/27310 (11.7%)]
[6400/27310 (23.4%)]
[9600/27310 (35.1%)]
[12800/27310 (46.8%)]
[16000/27310 (58.5%)]
[19200/27310 (70.3%)]
[22400/27310 (82.0%)]
[25600/27310 (93.7%)]
Avg val loss: 0.00684044
Train Fold:2/4  Ep: 1/3 [0/81928 (0%)]	 Loss: 0.007113	 Remain : 0:00:00 
Train Fold:2/4  Ep: 1/3 [3200/81928 (4%)]	 Loss: 0.006935	 Remain : 0:33:19 
Train Fold:2/4  Ep: 1/3 [6400/81928 (8%)]	 Loss: 0.007396	 Remain : 0:31:58 
Train Fold:2/4  Ep: 1/3 [9600/81928 (12%)]	 Loss: 0.007345	 Remain : 0:30:37 
Train Fold:2/4  Ep: 1/3 [12800/81928 (16%)]	 Loss: 0.007593	 Remain : 0:29:16 
Train Fold:2/4  Ep: 1/3 [16000/81928 (20%)]	 Loss: 0.006798	 Remain : 0:27:54 
Train Fold:2/4  Ep: 1/3 [19200/81928 (23%)]	 Loss: 0.006385	 Remain : 0:26:33 
Train Fold:2/4  Ep: 1/3 [22400/81928 (27%)]	 Loss: 0.006397	 Remain : 0:25:12 
Train Fold:2/4  Ep: 1/3 [25600/81928 (31%)]	 Loss: 0.007154	 Remain : 0:23:51 
Train Fold:2/4  Ep: 1/3 [28800/81928 (35%)]	 Loss: 0.007962	 Remain : 0:22:29 
Train Fold:2/4  Ep: 1/3 [32000/81928 (39%)]	 Loss: 0.005916	 Remain : 0:21:08 
Train Fold:2/4  Ep: 1/3 [35200/81928 (43%)]	 Loss: 0.007694	 Remain : 0:19:47 
Train Fold:2/4  Ep: 1/3 [38400/81928 (47%)]	 Loss: 0.006579	 Remain : 0:18:25 
Train Fold:2/4  Ep: 1/3 [41600/81928 (51%)]	 Loss: 0.006292	 Remain : 0:17:04 
Train Fold:2/4  Ep: 1/3 [44800/81928 (55%)]	 Loss: 0.005437	 Remain : 0:15:43 
Train Fold:2/4  Ep: 1/3 [48000/81928 (59%)]	 Loss: 0.007329	 Remain : 0:14:22 
Train Fold:2/4  Ep: 1/3 [51200/81928 (62%)]	 Loss: 0.005908	 Remain : 0:13:00 
Train Fold:2/4  Ep: 1/3 [54400/81928 (66%)]	 Loss: 0.005664	 Remain : 0:11:39 
Train Fold:2/4  Ep: 1/3 [57600/81928 (70%)]	 Loss: 0.006065	 Remain : 0:10:18 
Train Fold:2/4  Ep: 1/3 [60800/81928 (74%)]	 Loss: 0.005132	 Remain : 0:08:57 
Train Fold:2/4  Ep: 1/3 [64000/81928 (78%)]	 Loss: 0.004263	 Remain : 0:07:35 
Train Fold:2/4  Ep: 1/3 [67200/81928 (82%)]	 Loss: 0.006351	 Remain : 0:06:14 
Train Fold:2/4  Ep: 1/3 [70400/81928 (86%)]	 Loss: 0.006126	 Remain : 0:04:53 
Train Fold:2/4  Ep: 1/3 [73600/81928 (90%)]	 Loss: 0.006647	 Remain : 0:03:32 
Train Fold:2/4  Ep: 1/3 [76800/81928 (94%)]	 Loss: 0.004187	 Remain : 0:02:10 
Train Fold:2/4  Ep: 1/3 [80000/81928 (98%)]	 Loss: 0.004568	 Remain : 0:00:49 
Finish
[0/27309 (0.0%)]
[3200/27309 (11.7%)]
[6400/27309 (23.4%)]
[9600/27309 (35.1%)]
[12800/27309 (46.8%)]
[16000/27309 (58.5%)]
[19200/27309 (70.3%)]
[22400/27309 (82.0%)]
[25600/27309 (93.7%)]
Avg val loss: 0.00609191
Train Fold:2/4  Ep: 2/3 [0/81928 (0%)]	 Loss: 0.005269	 Remain : 0:00:00 
Train Fold:2/4  Ep: 2/3 [3200/81928 (4%)]	 Loss: 0.005173	 Remain : 0:31:58 
Train Fold:2/4  Ep: 2/3 [6400/81928 (8%)]	 Loss: 0.005312	 Remain : 0:30:40 
Train Fold:2/4  Ep: 2/3 [9600/81928 (12%)]	 Loss: 0.005648	 Remain : 0:29:22 
Train Fold:2/4  Ep: 2/3 [12800/81928 (16%)]	 Loss: 0.005080	 Remain : 0:28:04 
Train Fold:2/4  Ep: 2/3 [16000/81928 (20%)]	 Loss: 0.005446	 Remain : 0:26:46 
Train Fold:2/4  Ep: 2/3 [19200/81928 (23%)]	 Loss: 0.006162	 Remain : 0:25:28 
Train Fold:2/4  Ep: 2/3 [22400/81928 (27%)]	 Loss: 0.005460	 Remain : 0:24:10 
Train Fold:2/4  Ep: 2/3 [25600/81928 (31%)]	 Loss: 0.005031	 Remain : 0:22:52 
Train Fold:2/4  Ep: 2/3 [28800/81928 (35%)]	 Loss: 0.004711	 Remain : 0:21:34 
Train Fold:2/4  Ep: 2/3 [32000/81928 (39%)]	 Loss: 0.004765	 Remain : 0:20:16 
Train Fold:2/4  Ep: 2/3 [35200/81928 (43%)]	 Loss: 0.005708	 Remain : 0:18:58 
Train Fold:2/4  Ep: 2/3 [38400/81928 (47%)]	 Loss: 0.005967	 Remain : 0:17:40 
Train Fold:2/4  Ep: 2/3 [41600/81928 (51%)]	 Loss: 0.005343	 Remain : 0:16:22 
Train Fold:2/4  Ep: 2/3 [44800/81928 (55%)]	 Loss: 0.006640	 Remain : 0:15:04 
Train Fold:2/4  Ep: 2/3 [48000/81928 (59%)]	 Loss: 0.004733	 Remain : 0:13:46 
Train Fold:2/4  Ep: 2/3 [51200/81928 (62%)]	 Loss: 0.006264	 Remain : 0:12:28 
Train Fold:2/4  Ep: 2/3 [54400/81928 (66%)]	 Loss: 0.005437	 Remain : 0:11:11 
Train Fold:2/4  Ep: 2/3 [57600/81928 (70%)]	 Loss: 0.005551	 Remain : 0:09:53 
Train Fold:2/4  Ep: 2/3 [60800/81928 (74%)]	 Loss: 0.004954	 Remain : 0:08:35 
Train Fold:2/4  Ep: 2/3 [64000/81928 (78%)]	 Loss: 0.006316	 Remain : 0:07:17 
Train Fold:2/4  Ep: 2/3 [67200/81928 (82%)]	 Loss: 0.004692	 Remain : 0:05:59 
Train Fold:2/4  Ep: 2/3 [70400/81928 (86%)]	 Loss: 0.004600	 Remain : 0:04:41 
Train Fold:2/4  Ep: 2/3 [73600/81928 (90%)]	 Loss: 0.005306	 Remain : 0:03:23 
Train Fold:2/4  Ep: 2/3 [76800/81928 (94%)]	 Loss: 0.005180	 Remain : 0:02:05 
Train Fold:2/4  Ep: 2/3 [80000/81928 (98%)]	 Loss: 0.004779	 Remain : 0:00:47 
Finish
[0/27309 (0.0%)]
[3200/27309 (11.7%)]
[6400/27309 (23.4%)]
[9600/27309 (35.1%)]
[12800/27309 (46.8%)]
[16000/27309 (58.5%)]
[19200/27309 (70.3%)]
[22400/27309 (82.0%)]
[25600/27309 (93.7%)]
Avg val loss: 0.00608531
Train Fold:2/4  Ep: 3/3 [0/81928 (0%)]	 Loss: 0.004630	 Remain : 0:00:00 
Train Fold:2/4  Ep: 3/3 [3200/81928 (4%)]	 Loss: 0.006614	 Remain : 0:31:53 
Train Fold:2/4  Ep: 3/3 [6400/81928 (8%)]	 Loss: 0.005903	 Remain : 0:30:35 
Train Fold:2/4  Ep: 3/3 [9600/81928 (12%)]	 Loss: 0.005676	 Remain : 0:29:18 
Train Fold:2/4  Ep: 3/3 [12800/81928 (16%)]	 Loss: 0.005094	 Remain : 0:28:00 
Train Fold:2/4  Ep: 3/3 [16000/81928 (20%)]	 Loss: 0.006015	 Remain : 0:26:42 
Train Fold:2/4  Ep: 3/3 [19200/81928 (23%)]	 Loss: 0.006290	 Remain : 0:25:24 
Train Fold:2/4  Ep: 3/3 [22400/81928 (27%)]	 Loss: 0.004792	 Remain : 0:24:07 
Train Fold:2/4  Ep: 3/3 [25600/81928 (31%)]	 Loss: 0.005893	 Remain : 0:22:49 
Train Fold:2/4  Ep: 3/3 [28800/81928 (35%)]	 Loss: 0.004876	 Remain : 0:21:31 
Train Fold:2/4  Ep: 3/3 [32000/81928 (39%)]	 Loss: 0.005977	 Remain : 0:20:13 
Train Fold:2/4  Ep: 3/3 [35200/81928 (43%)]	 Loss: 0.007342	 Remain : 0:18:56 
Train Fold:2/4  Ep: 3/3 [38400/81928 (47%)]	 Loss: 0.006110	 Remain : 0:17:38 
Train Fold:2/4  Ep: 3/3 [41600/81928 (51%)]	 Loss: 0.006217	 Remain : 0:16:20 
Train Fold:2/4  Ep: 3/3 [44800/81928 (55%)]	 Loss: 0.004839	 Remain : 0:15:02 
Train Fold:2/4  Ep: 3/3 [48000/81928 (59%)]	 Loss: 0.005966	 Remain : 0:13:45 
Train Fold:2/4  Ep: 3/3 [51200/81928 (62%)]	 Loss: 0.004647	 Remain : 0:12:27 
Train Fold:2/4  Ep: 3/3 [54400/81928 (66%)]	 Loss: 0.005393	 Remain : 0:11:09 
Train Fold:2/4  Ep: 3/3 [57600/81928 (70%)]	 Loss: 0.004843	 Remain : 0:09:51 
Train Fold:2/4  Ep: 3/3 [60800/81928 (74%)]	 Loss: 0.005264	 Remain : 0:08:33 
Train Fold:2/4  Ep: 3/3 [64000/81928 (78%)]	 Loss: 0.005715	 Remain : 0:07:16 
Train Fold:2/4  Ep: 3/3 [67200/81928 (82%)]	 Loss: 0.005016	 Remain : 0:05:58 
Train Fold:2/4  Ep: 3/3 [70400/81928 (86%)]	 Loss: 0.006380	 Remain : 0:04:40 
Train Fold:2/4  Ep: 3/3 [73600/81928 (90%)]	 Loss: 0.005550	 Remain : 0:03:22 
Train Fold:2/4  Ep: 3/3 [76800/81928 (94%)]	 Loss: 0.006278	 Remain : 0:02:05 
Train Fold:2/4  Ep: 3/3 [80000/81928 (98%)]	 Loss: 0.005831	 Remain : 0:00:47 
Finish
[0/27309 (0.0%)]
[3200/27309 (11.7%)]
[6400/27309 (23.4%)]
[9600/27309 (35.1%)]
[12800/27309 (46.8%)]
[16000/27309 (58.5%)]
[19200/27309 (70.3%)]
[22400/27309 (82.0%)]
[25600/27309 (93.7%)]
Avg val loss: 0.00609392
Train Fold:3/4  Ep: 1/3 [0/81928 (0%)]	 Loss: 0.006293	 Remain : 0:00:00 
Train Fold:3/4  Ep: 1/3 [3200/81928 (4%)]	 Loss: 0.005850	 Remain : 0:33:18 
Train Fold:3/4  Ep: 1/3 [6400/81928 (8%)]	 Loss: 0.004770	 Remain : 0:31:57 
Train Fold:3/4  Ep: 1/3 [9600/81928 (12%)]	 Loss: 0.004487	 Remain : 0:30:36 
Train Fold:3/4  Ep: 1/3 [12800/81928 (16%)]	 Loss: 0.004694	 Remain : 0:29:15 
Train Fold:3/4  Ep: 1/3 [16000/81928 (20%)]	 Loss: 0.004243	 Remain : 0:27:53 
Train Fold:3/4  Ep: 1/3 [19200/81928 (23%)]	 Loss: 0.006256	 Remain : 0:26:32 
Train Fold:3/4  Ep: 1/3 [22400/81928 (27%)]	 Loss: 0.005965	 Remain : 0:25:11 
Train Fold:3/4  Ep: 1/3 [25600/81928 (31%)]	 Loss: 0.004864	 Remain : 0:23:50 
Train Fold:3/4  Ep: 1/3 [28800/81928 (35%)]	 Loss: 0.005645	 Remain : 0:22:28 
Train Fold:3/4  Ep: 1/3 [32000/81928 (39%)]	 Loss: 0.006038	 Remain : 0:21:07 
Train Fold:3/4  Ep: 1/3 [35200/81928 (43%)]	 Loss: 0.005668	 Remain : 0:19:46 
Train Fold:3/4  Ep: 1/3 [38400/81928 (47%)]	 Loss: 0.007684	 Remain : 0:18:25 
Train Fold:3/4  Ep: 1/3 [41600/81928 (51%)]	 Loss: 0.005893	 Remain : 0:17:04 
Train Fold:3/4  Ep: 1/3 [44800/81928 (55%)]	 Loss: 0.004278	 Remain : 0:15:42 
Train Fold:3/4  Ep: 1/3 [48000/81928 (59%)]	 Loss: 0.004922	 Remain : 0:14:21 
Train Fold:3/4  Ep: 1/3 [51200/81928 (62%)]	 Loss: 0.005632	 Remain : 0:13:00 
Train Fold:3/4  Ep: 1/3 [54400/81928 (66%)]	 Loss: 0.005661	 Remain : 0:11:39 
Train Fold:3/4  Ep: 1/3 [57600/81928 (70%)]	 Loss: 0.005354	 Remain : 0:10:18 
Train Fold:3/4  Ep: 1/3 [60800/81928 (74%)]	 Loss: 0.005077	 Remain : 0:08:56 
Train Fold:3/4  Ep: 1/3 [64000/81928 (78%)]	 Loss: 0.006033	 Remain : 0:07:35 
Train Fold:3/4  Ep: 1/3 [67200/81928 (82%)]	 Loss: 0.006080	 Remain : 0:06:14 
Train Fold:3/4  Ep: 1/3 [70400/81928 (86%)]	 Loss: 0.006449	 Remain : 0:04:53 
Train Fold:3/4  Ep: 1/3 [73600/81928 (90%)]	 Loss: 0.006409	 Remain : 0:03:31 
Train Fold:3/4  Ep: 1/3 [76800/81928 (94%)]	 Loss: 0.007354	 Remain : 0:02:10 
Train Fold:3/4  Ep: 1/3 [80000/81928 (98%)]	 Loss: 0.006782	 Remain : 0:00:49 
Finish
[0/27309 (0.0%)]
[3200/27309 (11.7%)]
[6400/27309 (23.4%)]
[9600/27309 (35.1%)]
[12800/27309 (46.8%)]
[16000/27309 (58.5%)]
[19200/27309 (70.3%)]
[22400/27309 (82.0%)]
[25600/27309 (93.7%)]
Avg val loss: 0.00538189
Train Fold:3/4  Ep: 2/3 [0/81928 (0%)]	 Loss: 0.005512	 Remain : 0:00:00 
Train Fold:3/4  Ep: 2/3 [3200/81928 (4%)]	 Loss: 0.005343	 Remain : 0:31:41 
Train Fold:3/4  Ep: 2/3 [6400/81928 (8%)]	 Loss: 0.004435	 Remain : 0:30:24 
Train Fold:3/4  Ep: 2/3 [9600/81928 (12%)]	 Loss: 0.004132	 Remain : 0:29:07 
Train Fold:3/4  Ep: 2/3 [12800/81928 (16%)]	 Loss: 0.005336	 Remain : 0:27:50 
Train Fold:3/4  Ep: 2/3 [16000/81928 (20%)]	 Loss: 0.005147	 Remain : 0:26:32 
Train Fold:3/4  Ep: 2/3 [19200/81928 (23%)]	 Loss: 0.004751	 Remain : 0:25:15 
Train Fold:3/4  Ep: 2/3 [22400/81928 (27%)]	 Loss: 0.003634	 Remain : 0:23:58 
Train Fold:3/4  Ep: 2/3 [25600/81928 (31%)]	 Loss: 0.005480	 Remain : 0:22:40 
Train Fold:3/4  Ep: 2/3 [28800/81928 (35%)]	 Loss: 0.004121	 Remain : 0:21:23 
Train Fold:3/4  Ep: 2/3 [32000/81928 (39%)]	 Loss: 0.004612	 Remain : 0:20:06 
Train Fold:3/4  Ep: 2/3 [35200/81928 (43%)]	 Loss: 0.005434	 Remain : 0:18:49 
Train Fold:3/4  Ep: 2/3 [38400/81928 (47%)]	 Loss: 0.004852	 Remain : 0:17:31 
Train Fold:3/4  Ep: 2/3 [41600/81928 (51%)]	 Loss: 0.003999	 Remain : 0:16:14 
Train Fold:3/4  Ep: 2/3 [44800/81928 (55%)]	 Loss: 0.004804	 Remain : 0:14:57 
Train Fold:3/4  Ep: 2/3 [48000/81928 (59%)]	 Loss: 0.004819	 Remain : 0:13:39 
Train Fold:3/4  Ep: 2/3 [51200/81928 (62%)]	 Loss: 0.004479	 Remain : 0:12:22 
Train Fold:3/4  Ep: 2/3 [54400/81928 (66%)]	 Loss: 0.003792	 Remain : 0:11:05 
Train Fold:3/4  Ep: 2/3 [57600/81928 (70%)]	 Loss: 0.005248	 Remain : 0:09:48 
Train Fold:3/4  Ep: 2/3 [60800/81928 (74%)]	 Loss: 0.004438	 Remain : 0:08:30 
Train Fold:3/4  Ep: 2/3 [64000/81928 (78%)]	 Loss: 0.004469	 Remain : 0:07:13 
Train Fold:3/4  Ep: 2/3 [67200/81928 (82%)]	 Loss: 0.004920	 Remain : 0:05:56 
Train Fold:3/4  Ep: 2/3 [70400/81928 (86%)]	 Loss: 0.004728	 Remain : 0:04:38 
Train Fold:3/4  Ep: 2/3 [73600/81928 (90%)]	 Loss: 0.005466	 Remain : 0:03:21 
Train Fold:3/4  Ep: 2/3 [76800/81928 (94%)]	 Loss: 0.004547	 Remain : 0:02:04 
Train Fold:3/4  Ep: 2/3 [80000/81928 (98%)]	 Loss: 0.003625	 Remain : 0:00:47 
Finish
[0/27309 (0.0%)]
[3200/27309 (11.7%)]
[6400/27309 (23.4%)]
[9600/27309 (35.1%)]
[12800/27309 (46.8%)]
[16000/27309 (58.5%)]
[19200/27309 (70.3%)]
[22400/27309 (82.0%)]
[25600/27309 (93.7%)]
Avg val loss: 0.00536613
Train Fold:3/4  Ep: 3/3 [0/81928 (0%)]	 Loss: 0.004166	 Remain : 0:00:00 
Train Fold:3/4  Ep: 3/3 [3200/81928 (4%)]	 Loss: 0.004029	 Remain : 0:31:51 
Train Fold:3/4  Ep: 3/3 [6400/81928 (8%)]	 Loss: 0.004879	 Remain : 0:30:33 
Train Fold:3/4  Ep: 3/3 [9600/81928 (12%)]	 Loss: 0.004718	 Remain : 0:29:16 
Train Fold:3/4  Ep: 3/3 [12800/81928 (16%)]	 Loss: 0.005013	 Remain : 0:27:58 
Train Fold:3/4  Ep: 3/3 [16000/81928 (20%)]	 Loss: 0.005278	 Remain : 0:26:40 
Train Fold:3/4  Ep: 3/3 [19200/81928 (23%)]	 Loss: 0.004373	 Remain : 0:25:23 
Train Fold:3/4  Ep: 3/3 [22400/81928 (27%)]	 Loss: 0.004873	 Remain : 0:24:05 
Train Fold:3/4  Ep: 3/3 [25600/81928 (31%)]	 Loss: 0.004652	 Remain : 0:22:47 
Train Fold:3/4  Ep: 3/3 [28800/81928 (35%)]	 Loss: 0.004189	 Remain : 0:21:30 
Train Fold:3/4  Ep: 3/3 [32000/81928 (39%)]	 Loss: 0.004423	 Remain : 0:20:12 
Train Fold:3/4  Ep: 3/3 [35200/81928 (43%)]	 Loss: 0.005896	 Remain : 0:18:54 
Train Fold:3/4  Ep: 3/3 [38400/81928 (47%)]	 Loss: 0.004217	 Remain : 0:17:37 
Train Fold:3/4  Ep: 3/3 [41600/81928 (51%)]	 Loss: 0.005019	 Remain : 0:16:19 
Train Fold:3/4  Ep: 3/3 [44800/81928 (55%)]	 Loss: 0.004166	 Remain : 0:15:01 
Train Fold:3/4  Ep: 3/3 [48000/81928 (59%)]	 Loss: 0.004917	 Remain : 0:13:44 
Train Fold:3/4  Ep: 3/3 [51200/81928 (62%)]	 Loss: 0.004275	 Remain : 0:12:26 
Train Fold:3/4  Ep: 3/3 [54400/81928 (66%)]	 Loss: 0.003676	 Remain : 0:11:08 
Train Fold:3/4  Ep: 3/3 [57600/81928 (70%)]	 Loss: 0.003897	 Remain : 0:09:51 
Train Fold:3/4  Ep: 3/3 [60800/81928 (74%)]	 Loss: 0.005373	 Remain : 0:08:33 
Train Fold:3/4  Ep: 3/3 [64000/81928 (78%)]	 Loss: 0.005255	 Remain : 0:07:15 
Train Fold:3/4  Ep: 3/3 [67200/81928 (82%)]	 Loss: 0.005146	 Remain : 0:05:58 
Train Fold:3/4  Ep: 3/3 [70400/81928 (86%)]	 Loss: 0.003847	 Remain : 0:04:40 
Train Fold:3/4  Ep: 3/3 [73600/81928 (90%)]	 Loss: 0.004856	 Remain : 0:03:22 
Train Fold:3/4  Ep: 3/3 [76800/81928 (94%)]	 Loss: 0.003601	 Remain : 0:02:05 
Train Fold:3/4  Ep: 3/3 [80000/81928 (98%)]	 Loss: 0.004764	 Remain : 0:00:47 
Finish
[0/27309 (0.0%)]
[3200/27309 (11.7%)]
[6400/27309 (23.4%)]
[9600/27309 (35.1%)]
[12800/27309 (46.8%)]
[16000/27309 (58.5%)]
[19200/27309 (70.3%)]
[22400/27309 (82.0%)]
[25600/27309 (93.7%)]
Avg val loss: 0.00536149
Train Fold:4/4  Ep: 1/3 [0/81928 (0%)]	 Loss: 0.005960	 Remain : 0:00:00 
Train Fold:4/4  Ep: 1/3 [3200/81928 (4%)]	 Loss: 0.005302	 Remain : 0:33:11 
Train Fold:4/4  Ep: 1/3 [6400/81928 (8%)]	 Loss: 0.005185	 Remain : 0:31:50 
Train Fold:4/4  Ep: 1/3 [9600/81928 (12%)]	 Loss: 0.005437	 Remain : 0:30:29 
Train Fold:4/4  Ep: 1/3 [12800/81928 (16%)]	 Loss: 0.005124	 Remain : 0:29:08 
Train Fold:4/4  Ep: 1/3 [16000/81928 (20%)]	 Loss: 0.005892	 Remain : 0:27:47 
Train Fold:4/4  Ep: 1/3 [19200/81928 (23%)]	 Loss: 0.005511	 Remain : 0:26:26 
Train Fold:4/4  Ep: 1/3 [22400/81928 (27%)]	 Loss: 0.004921	 Remain : 0:25:05 
Train Fold:4/4  Ep: 1/3 [25600/81928 (31%)]	 Loss: 0.005326	 Remain : 0:23:44 
Train Fold:4/4  Ep: 1/3 [28800/81928 (35%)]	 Loss: 0.006013	 Remain : 0:22:24 
Train Fold:4/4  Ep: 1/3 [32000/81928 (39%)]	 Loss: 0.006327	 Remain : 0:21:03 
Train Fold:4/4  Ep: 1/3 [35200/81928 (43%)]	 Loss: 0.004353	 Remain : 0:19:42 
Train Fold:4/4  Ep: 1/3 [38400/81928 (47%)]	 Loss: 0.005550	 Remain : 0:18:21 
Train Fold:4/4  Ep: 1/3 [41600/81928 (51%)]	 Loss: 0.004895	 Remain : 0:17:00 
Train Fold:4/4  Ep: 1/3 [44800/81928 (55%)]	 Loss: 0.005051	 Remain : 0:15:39 
Train Fold:4/4  Ep: 1/3 [48000/81928 (59%)]	 Loss: 0.004360	 Remain : 0:14:18 
Train Fold:4/4  Ep: 1/3 [51200/81928 (62%)]	 Loss: 0.005455	 Remain : 0:12:57 
Train Fold:4/4  Ep: 1/3 [54400/81928 (66%)]	 Loss: 0.005668	 Remain : 0:11:36 
Train Fold:4/4  Ep: 1/3 [57600/81928 (70%)]	 Loss: 0.004801	 Remain : 0:10:15 
Train Fold:4/4  Ep: 1/3 [60800/81928 (74%)]	 Loss: 0.005190	 Remain : 0:08:54 
Train Fold:4/4  Ep: 1/3 [64000/81928 (78%)]	 Loss: 0.005228	 Remain : 0:07:33 
Train Fold:4/4  Ep: 1/3 [67200/81928 (82%)]	 Loss: 0.004312	 Remain : 0:06:13 
Train Fold:4/4  Ep: 1/3 [70400/81928 (86%)]	 Loss: 0.004637	 Remain : 0:04:52 
Train Fold:4/4  Ep: 1/3 [73600/81928 (90%)]	 Loss: 0.005041	 Remain : 0:03:31 
Train Fold:4/4  Ep: 1/3 [76800/81928 (94%)]	 Loss: 0.004934	 Remain : 0:02:10 
Train Fold:4/4  Ep: 1/3 [80000/81928 (98%)]	 Loss: 0.005781	 Remain : 0:00:49 
Finish
[0/27309 (0.0%)]
[3200/27309 (11.7%)]
[6400/27309 (23.4%)]
[9600/27309 (35.1%)]
[12800/27309 (46.8%)]
[16000/27309 (58.5%)]
[19200/27309 (70.3%)]
[22400/27309 (82.0%)]
[25600/27309 (93.7%)]
Avg val loss: 0.00468685
Train Fold:4/4  Ep: 2/3 [0/81928 (0%)]	 Loss: 0.004073	 Remain : 0:00:00 
Train Fold:4/4  Ep: 2/3 [3200/81928 (4%)]	 Loss: 0.005640	 Remain : 0:31:58 
Train Fold:4/4  Ep: 2/3 [6400/81928 (8%)]	 Loss: 0.003811	 Remain : 0:30:40 
Train Fold:4/4  Ep: 2/3 [9600/81928 (12%)]	 Loss: 0.004413	 Remain : 0:29:22 
Train Fold:4/4  Ep: 2/3 [12800/81928 (16%)]	 Loss: 0.004090	 Remain : 0:28:04 
Train Fold:4/4  Ep: 2/3 [16000/81928 (20%)]	 Loss: 0.003872	 Remain : 0:26:46 
Train Fold:4/4  Ep: 2/3 [19200/81928 (23%)]	 Loss: 0.004307	 Remain : 0:25:28 
Train Fold:4/4  Ep: 2/3 [22400/81928 (27%)]	 Loss: 0.004407	 Remain : 0:24:10 
Train Fold:4/4  Ep: 2/3 [25600/81928 (31%)]	 Loss: 0.004245	 Remain : 0:22:52 
Train Fold:4/4  Ep: 2/3 [28800/81928 (35%)]	 Loss: 0.003776	 Remain : 0:21:34 
Train Fold:4/4  Ep: 2/3 [32000/81928 (39%)]	 Loss: 0.003581	 Remain : 0:20:16 
Train Fold:4/4  Ep: 2/3 [35200/81928 (43%)]	 Loss: 0.004433	 Remain : 0:18:58 
Train Fold:4/4  Ep: 2/3 [38400/81928 (47%)]	 Loss: 0.004450	 Remain : 0:17:40 
Train Fold:4/4  Ep: 2/3 [41600/81928 (51%)]	 Loss: 0.004098	 Remain : 0:16:22 
Train Fold:4/4  Ep: 2/3 [44800/81928 (55%)]	 Loss: 0.003461	 Remain : 0:15:04 
Train Fold:4/4  Ep: 2/3 [48000/81928 (59%)]	 Loss: 0.003826	 Remain : 0:13:47 
Train Fold:4/4  Ep: 2/3 [51200/81928 (62%)]	 Loss: 0.003750	 Remain : 0:12:29 
Train Fold:4/4  Ep: 2/3 [54400/81928 (66%)]	 Loss: 0.005291	 Remain : 0:11:11 
Train Fold:4/4  Ep: 2/3 [57600/81928 (70%)]	 Loss: 0.004572	 Remain : 0:09:53 
Train Fold:4/4  Ep: 2/3 [60800/81928 (74%)]	 Loss: 0.004117	 Remain : 0:08:35 
Train Fold:4/4  Ep: 2/3 [64000/81928 (78%)]	 Loss: 0.003736	 Remain : 0:07:17 
Train Fold:4/4  Ep: 2/3 [67200/81928 (82%)]	 Loss: 0.004272	 Remain : 0:05:59 
Train Fold:4/4  Ep: 2/3 [70400/81928 (86%)]	 Loss: 0.004261	 Remain : 0:04:41 
Train Fold:4/4  Ep: 2/3 [73600/81928 (90%)]	 Loss: 0.004175	 Remain : 0:03:23 
Train Fold:4/4  Ep: 2/3 [76800/81928 (94%)]	 Loss: 0.005109	 Remain : 0:02:05 
Train Fold:4/4  Ep: 2/3 [80000/81928 (98%)]	 Loss: 0.004151	 Remain : 0:00:47 
Finish
[0/27309 (0.0%)]
[3200/27309 (11.7%)]
[6400/27309 (23.4%)]
[9600/27309 (35.1%)]
[12800/27309 (46.8%)]
[16000/27309 (58.5%)]
[19200/27309 (70.3%)]
[22400/27309 (82.0%)]
[25600/27309 (93.7%)]
Avg val loss: 0.00469111
Train Fold:4/4  Ep: 3/3 [0/81928 (0%)]	 Loss: 0.003701	 Remain : 0:00:00 
Train Fold:4/4  Ep: 3/3 [3200/81928 (4%)]	 Loss: 0.003713	 Remain : 0:31:57 
Train Fold:4/4  Ep: 3/3 [6400/81928 (8%)]	 Loss: 0.004336	 Remain : 0:30:40 
Train Fold:4/4  Ep: 3/3 [9600/81928 (12%)]	 Loss: 0.003428	 Remain : 0:29:22 
Train Fold:4/4  Ep: 3/3 [12800/81928 (16%)]	 Loss: 0.003651	 Remain : 0:28:04 
Train Fold:4/4  Ep: 3/3 [16000/81928 (20%)]	 Loss: 0.004642	 Remain : 0:26:46 
Train Fold:4/4  Ep: 3/3 [19200/81928 (23%)]	 Loss: 0.004406	 Remain : 0:25:28 
Train Fold:4/4  Ep: 3/3 [22400/81928 (27%)]	 Loss: 0.003148	 Remain : 0:24:10 
Train Fold:4/4  Ep: 3/3 [25600/81928 (31%)]	 Loss: 0.005396	 Remain : 0:22:52 
Train Fold:4/4  Ep: 3/3 [28800/81928 (35%)]	 Loss: 0.004629	 Remain : 0:21:34 
Train Fold:4/4  Ep: 3/3 [32000/81928 (39%)]	 Loss: 0.003810	 Remain : 0:20:16 
Train Fold:4/4  Ep: 3/3 [35200/81928 (43%)]	 Loss: 0.003808	 Remain : 0:18:58 
Train Fold:4/4  Ep: 3/3 [38400/81928 (47%)]	 Loss: 0.006448	 Remain : 0:17:40 
Train Fold:4/4  Ep: 3/3 [41600/81928 (51%)]	 Loss: 0.004178	 Remain : 0:16:22 
Train Fold:4/4  Ep: 3/3 [44800/81928 (55%)]	 Loss: 0.004317	 Remain : 0:15:04 
Train Fold:4/4  Ep: 3/3 [48000/81928 (59%)]	 Loss: 0.003719	 Remain : 0:13:46 
Train Fold:4/4  Ep: 3/3 [51200/81928 (62%)]	 Loss: 0.004984	 Remain : 0:12:28 
Train Fold:4/4  Ep: 3/3 [54400/81928 (66%)]	 Loss: 0.004656	 Remain : 0:11:11 
Train Fold:4/4  Ep: 3/3 [57600/81928 (70%)]	 Loss: 0.004213	 Remain : 0:09:53 
Train Fold:4/4  Ep: 3/3 [60800/81928 (74%)]	 Loss: 0.005300	 Remain : 0:08:35 
Train Fold:4/4  Ep: 3/3 [64000/81928 (78%)]	 Loss: 0.004308	 Remain : 0:07:17 
Train Fold:4/4  Ep: 3/3 [67200/81928 (82%)]	 Loss: 0.003197	 Remain : 0:05:59 
Train Fold:4/4  Ep: 3/3 [70400/81928 (86%)]	 Loss: 0.004019	 Remain : 0:04:41 
Train Fold:4/4  Ep: 3/3 [73600/81928 (90%)]	 Loss: 0.004233	 Remain : 0:03:23 
Train Fold:4/4  Ep: 3/3 [76800/81928 (94%)]	 Loss: 0.003452	 Remain : 0:02:05 
Train Fold:4/4  Ep: 3/3 [80000/81928 (98%)]	 Loss: 0.003703	 Remain : 0:00:47 
Finish
[0/27309 (0.0%)]
[3200/27309 (11.7%)]
[6400/27309 (23.4%)]
[9600/27309 (35.1%)]
[12800/27309 (46.8%)]
[16000/27309 (58.5%)]
[19200/27309 (70.3%)]
[22400/27309 (82.0%)]
[25600/27309 (93.7%)]
Avg val loss: 0.00468680
In [33]:
torch.cuda.empty_cache()
!nvidia-smi
Thu Jun  6 13:37:46 2019       
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 410.104      Driver Version: 410.104      CUDA Version: 10.0     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  Tesla P100-PCIE...  On   | 00000000:00:04.0 Off |                    0 |
| N/A   72C    P0    52W / 250W |   2319MiB / 16280MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+
                                                                               
+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU       PID   Type   Process name                             Usage      |
|=============================================================================|
+-----------------------------------------------------------------------------+
In [34]:
torch.save(model, 'net.pkl')
#model = torch.load('net.pkl')
/opt/conda/lib/python3.6/site-packages/torch/serialization.py:251: UserWarning: Couldn't retrieve source code for container of type SENet. It won't be checked for correctness upon loading.
  "type " + obj.__name__ + ". It won't be checked "
/opt/conda/lib/python3.6/site-packages/torch/serialization.py:251: UserWarning: Couldn't retrieve source code for container of type SEResNeXtBottleneck. It won't be checked for correctness upon loading.
  "type " + obj.__name__ + ". It won't be checked "
/opt/conda/lib/python3.6/site-packages/torch/serialization.py:251: UserWarning: Couldn't retrieve source code for container of type SEModule. It won't be checked for correctness upon loading.
  "type " + obj.__name__ + ". It won't be checked "
In [35]:
def findPre(output,gate):
    a = ''
    output = np.array(output)
    m = np.max(output)

    for i in range(len(output)):
        #s = np.where(v[i] > 0.95, 1, 0)
        if output[i]>gate:
            #print(output[i])
            a = a + format(i)+' '
            
    #print(a)
    return a
    
In [36]:
def test(model,dloader,threshold):
    model = model.eval().cuda()
    #lengthD = len(dloader.dataset)
    ans = []
    out = []
    for step, (x,x2, y) in enumerate(dloader):
        data = Variable(x).cuda()
        #data2 = Variable(x2).cuda()
        #data = Variable(x)
        target = y
        #output = model(data,data2)
        output = model(data).detach()
        
        #data.cpu()   # batch x
        #data2.cpu()   # batch x
        #target.cpu()   # batch y
        #torch.cuda.empty_cache()
        
        v = output.cpu()
        v = torch.sigmoid(v)
        
        v = torch.sigmoid(v)
        v = np.array(v)
        v = preprocessing.minmax_scale(v, feature_range=(0,1),axis=1)
        for i in range(len(v)):
            out.append(np.where(v[i] > threshold, 1, 0))
            s = findPre(v[i],threshold)
            ans.append([target[i],s])
        if step %10 == 0:
            print('[{}/{} ({:.1f}%)]'.format(step * len(data), 
                                        len(dloader.dataset),
                                        100.*step/len(dloader)))
            
        data.cpu()
        #data2.detach()
        #target.detach()
        torch.cuda.empty_cache()
    print("Finish")
    return ans,out
    
In [37]:
from sklearn.metrics import fbeta_score

def makeScore(pre,ans):
    pre = np.array(pre)
    va = fbeta_score(y_pred=pre, y_true=ans, beta=2, average="samples")
    print("Score : {:.5f}".format(va))
    return va
    
In [38]:
def findThreshold():
    score = []
    candidates = np.arange(0, 1.0, 0.01)
    for th in candidates:
        print("Threshold : {:.2f}".format(th))
        _,pre = test(model = model,dloader = score_dataloader,threshold = th)
        #return pre
        score.append(makeScore(np.array(pre),np.array(train_a[:1000,1].tolist())))
        print("=============================")
    pm = np.array(score).argmax()
    best_th, best_score = candidates[pm], score[pm]
    return best_th, best_score
In [39]:
bt, bs = findThreshold()
print("Best Threshold : {:.2f}".format(bt))
print("Best Score : {:.5f}".format(bs))
Threshold : 0.00
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.01511
=============================
Threshold : 0.01
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.29233
=============================
Threshold : 0.02
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.39148
=============================
Threshold : 0.03
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.45917
=============================
Threshold : 0.04
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.50794
=============================
Threshold : 0.05
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.54631
=============================
Threshold : 0.06
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.57625
=============================
Threshold : 0.07
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.60462
=============================
Threshold : 0.08
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.62846
=============================
Threshold : 0.09
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.64642
=============================
Threshold : 0.10
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.66154
=============================
Threshold : 0.11
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.67493
=============================
Threshold : 0.12
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.68538
=============================
Threshold : 0.13
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.69447
=============================
Threshold : 0.14
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.70268
=============================
Threshold : 0.15
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.70790
=============================
Threshold : 0.16
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.71344
=============================
Threshold : 0.17
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.71840
=============================
Threshold : 0.18
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.72229
=============================
Threshold : 0.19
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.72623
=============================
Threshold : 0.20
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.72885
=============================
Threshold : 0.21
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.73023
=============================
Threshold : 0.22
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.73138
=============================
Threshold : 0.23
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.73402
=============================
Threshold : 0.24
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.73375
=============================
Threshold : 0.25
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.73270
=============================
Threshold : 0.26
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.73376
=============================
Threshold : 0.27
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.73335
=============================
Threshold : 0.28
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.73340
=============================
Threshold : 0.29
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.73223
=============================
Threshold : 0.30
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.73020
=============================
Threshold : 0.31
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.72994
=============================
Threshold : 0.32
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.72742
=============================
Threshold : 0.33
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.72775
=============================
Threshold : 0.34
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.72534
=============================
Threshold : 0.35
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.72407
=============================
Threshold : 0.36
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.72222
=============================
Threshold : 0.37
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.71899
=============================
Threshold : 0.38
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.71594
=============================
Threshold : 0.39
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.71260
=============================
Threshold : 0.40
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.71173
=============================
Threshold : 0.41
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.70698
=============================
Threshold : 0.42
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.70491
=============================
Threshold : 0.43
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.70255
=============================
Threshold : 0.44
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.70090
=============================
Threshold : 0.45
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.69749
=============================
Threshold : 0.46
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.69361
=============================
Threshold : 0.47
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.69020
=============================
Threshold : 0.48
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.68781
=============================
Threshold : 0.49
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.68581
=============================
Threshold : 0.50
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.68353
=============================
Threshold : 0.51
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.67962
=============================
Threshold : 0.52
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.67846
=============================
Threshold : 0.53
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.67356
=============================
Threshold : 0.54
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.66837
=============================
Threshold : 0.55
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.66639
=============================
Threshold : 0.56
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.66203
=============================
Threshold : 0.57
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.65946
=============================
Threshold : 0.58
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.65283
=============================
Threshold : 0.59
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.64980
=============================
Threshold : 0.60
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.64436
=============================
Threshold : 0.61
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.64231
=============================
Threshold : 0.62
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.63809
=============================
Threshold : 0.63
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.63450
=============================
Threshold : 0.64
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.63006
=============================
Threshold : 0.65
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.62418
=============================
Threshold : 0.66
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.61856
=============================
Threshold : 0.67
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.61514
=============================
Threshold : 0.68
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.61108
=============================
Threshold : 0.69
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.60495
=============================
Threshold : 0.70
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.60027
=============================
Threshold : 0.71
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.59499
=============================
Threshold : 0.72
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.58925
=============================
Threshold : 0.73
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.58460
=============================
Threshold : 0.74
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.58123
=============================
Threshold : 0.75
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.57591
=============================
Threshold : 0.76
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.57071
=============================
Threshold : 0.77
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.56683
=============================
Threshold : 0.78
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.56104
=============================
Threshold : 0.79
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.55624
=============================
Threshold : 0.80
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.55173
=============================
Threshold : 0.81
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.54789
=============================
Threshold : 0.82
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.54318
=============================
Threshold : 0.83
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.53695
=============================
Threshold : 0.84
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.53198
=============================
Threshold : 0.85
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.52234
=============================
Threshold : 0.86
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.51516
=============================
Threshold : 0.87
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.51070
=============================
Threshold : 0.88
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.50558
=============================
Threshold : 0.89
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.49854
=============================
Threshold : 0.90
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.48832
=============================
Threshold : 0.91
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.48163
=============================
Threshold : 0.92
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.47365
=============================
Threshold : 0.93
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.46476
=============================
Threshold : 0.94
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.45714
=============================
Threshold : 0.95
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.44776
=============================
Threshold : 0.96
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.44044
=============================
Threshold : 0.97
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.42996
=============================
Threshold : 0.98
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.41916
=============================
Threshold : 0.99
[0/1000 (0.0%)]
[320/1000 (31.2%)]
[640/1000 (62.5%)]
[960/1000 (93.8%)]
Finish
Score : 0.40338
=============================
Best Threshold : 0.23
Best Score : 0.73402
In [40]:
torch.cuda.empty_cache()
!nvidia-smi
Thu Jun  6 14:08:08 2019       
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 410.104      Driver Version: 410.104      CUDA Version: 10.0     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  Tesla P100-PCIE...  On   | 00000000:00:04.0 Off |                    0 |
| N/A   72C    P0    52W / 250W |   2319MiB / 16280MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+
                                                                               
+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU       PID   Type   Process name                             Usage      |
|=============================================================================|
+-----------------------------------------------------------------------------+
In [41]:
sub,_ = test(model = model,dloader = test_dataloader,threshold = bt)
[0/7443 (0.0%)]
[320/7443 (4.3%)]
[640/7443 (8.6%)]
[960/7443 (12.9%)]
[1280/7443 (17.2%)]
[1600/7443 (21.5%)]
[1920/7443 (25.8%)]
[2240/7443 (30.0%)]
[2560/7443 (34.3%)]
[2880/7443 (38.6%)]
[3200/7443 (42.9%)]
[3520/7443 (47.2%)]
[3840/7443 (51.5%)]
[4160/7443 (55.8%)]
[4480/7443 (60.1%)]
[4800/7443 (64.4%)]
[5120/7443 (68.7%)]
[5440/7443 (73.0%)]
[5760/7443 (77.3%)]
[6080/7443 (81.5%)]
[6400/7443 (85.8%)]
[6720/7443 (90.1%)]
[7040/7443 (94.4%)]
[7360/7443 (98.7%)]
Finish
In [42]:
sub =  pd.DataFrame(sub)
In [43]:
sub = sub.rename(index=str, columns={0: "id", 1: "attribute_ids"})
In [44]:
sub.head
Out[44]:
<bound method NDFrame.head of                     id                                      attribute_ids
0     10023b2cc4ed5f68                          223 337 344 369 766 1059 
1     100fbe75ed8fd887                                   93 231 369 1039 
2     101b627524a04f19                                79 182 728 784 961 
3     10234480c41284c6       51 147 480 483 725 738 776 813 830 923 1046 
4     1023b0e2636dcea8         147 156 227 322 477 616 813 954 1046 1092 
5      1039cd6cf85845c                                   13 405 903 1092 
6      103a5b3f83fbe88                              194 671 780 813 1092 
7     10413aaae8d6a9a2                                 147 813 1046 1092 
8     10423822b93a65ab  51 79 121 147 156 184 202 335 369 400 434 776 ...
9     1052bf702cb099f7                                   188 420 597 780 
10     10543c918a43a8d                  189 462 541 800 813 835 993 1092 
11    105c9a3453da79c3           121 145 161 188 304 464 493 671 840 872 
12    1060688bbf6eac87                      418 541 542 796 936 940 1072 
13    106a247caeabd15a                                          194 1034 
14    106e21606add59f3                                      737 813 1046 
15    107c38495881b6c9  51 110 147 156 185 212 283 554 584 671 744 746...
16    108815dd3752ab64                       335 377 420 477 489 554 612 
17    10943defdd5d5e89      121 188 420 477 612 668 738 744 784 813 1092 
18    10a39a78c44ef27c                         188 764 803 813 1092 1099 
19     10ab70df067bdb4                                        13 14 1056 
20    10b28e3de3566582                                   188 552 671 780 
21    10b32964331a6cc3                    99 161 733 734 784 813 961 996 
22    10b4562e7fa6f668                                           189 485 
23    10db1c338e1d822f                                      671 780 1059 
24    10e0c215f5f3084e                                       147 671 780 
25    10e95bead8e0b35b                                      776 830 1046 
26    1100d7b0f24fee88                      147 189 462 733 813 993 1092 
27    11099b321e8c7066                          156 485 744 813 835 1092 
28    110df388fd5c50e4                                           813 896 
29    113520ea0138f76d                   51 554 744 764 813 903 910 1092 
...                ...                                                ...
7413  ff2da1f0ed3e3ebe                                    13 477 671 780 
7414  ff3a9fa43f8eab9c                         188 420 428 597 1002 1099 
7415  ff44490e20740a19  79 121 161 184 204 236 438 450 703 720 738 813...
7416  ff481bb029678d5d                            51 393 489 584 671 780 
7417  ff4c3570fb7b90d3                           156 477 639 671 744 950 
7418  ff4f548d08414709                                      189 780 1034 
7419  ff668377a518ea5f  51 79 156 194 227 369 420 477 506 612 671 738 ...
7420  ff6a549b2d7a0e76  156 189 227 465 584 616 733 737 738 776 813 95...
7421  ff6ee1b37c8dc1ae           13 51 111 147 189 744 813 896 1046 1092 
7422  ff85460d6b853b49                             13 51 147 552 671 780 
7423   ff8721b85d1b5a5                                   13 405 896 1092 
7424  ff8bef7d0de52b31                                           189 768 
7425  ff92504c82c41e0f                           79 480 737 776 830 1046 
7426  ff9d4b77c124c9f2                                   147 597 671 835 
7427  ff9ddf70cb1c2674                           51 147 744 813 903 1092 
7428  ffaf8c3fe0b1d9b6                                    78 369 668 754 
7429  ffb61df4a6734772                                       313 420 962 
7430  ffb73f95b8721900                                           194 584 
7431  ffb937b55755323e                       51 156 671 744 758 813 1059 
7432  ffbcf8b91a8e8ce0                                           671 754 
7433  ffbf4849bde21b0a                  111 420 466 679 784 912 971 1037 
7434  ffc96e053345419d                              304 718 813 896 1092 
7435  ffcb16053099d795                              147 189 716 813 1092 
7436  ffcf745289465074                                  194 404 813 1092 
7437  ffd1372fe67e65f0                                          158 1039 
7438  ffd79eadf642221b                            45 189 718 813 896 903 
7439  ffd96986aa333f4d  70 147 189 418 541 542 543 579 753 813 835 949...
7440  ffe54b454396d97c          79 111 156 189 477 671 780 950 1034 1035 
7441  ffe7d7db4e4aa37f                                  189 671 768 1034 
7442  ffed0a4aca0d5457                                   304 349 487 758 

[7443 rows x 2 columns]>
In [45]:
sub.to_csv('submission.csv', index=False)