iMet Pytorch Starter

From: https://www.kaggle.com/hidehisaarai1213/imet-pytorch-starter

Author: kaggle masterのアライさん

Score: 0.474

Problem description

In this kernel, we are going to use Densenet121 pretrained model with Pytorch.

Libraries

In [1]:
import gc
import os
import sys
import time
import random
import logging
import datetime as dt

import numpy as np
import pandas as pd

import torch
import torch.nn as nn
import torch.optim as optim
import torch.utils.data as data
import torch.nn.functional as F
import torchvision as vision

from torch.optim.lr_scheduler import CosineAnnealingLR

from pathlib import Path
from PIL import Image
from contextlib import contextmanager

from joblib import Parallel, delayed
from tqdm import tqdm
from fastprogress import master_bar, progress_bar

from sklearn.model_selection import KFold
from sklearn.metrics import fbeta_score

torch.multiprocessing.set_start_method("spawn")

Utilities

In [2]:
@contextmanager
def timer(name="Main", logger=None):
    t0 = time.time()
    yield
    msg = f"[{name}] done in {time.time() - t0} s"
    if logger is not None:
        logger.info(msg)
    else:
        print(msg)
        

def get_logger(name="Main", tag="exp", log_dir="log/"):
    log_path = Path(log_dir)
    path = log_path / tag
    path.mkdir(exist_ok=True, parents=True)

    logger = logging.getLogger(name)
    logger.setLevel(logging.INFO)

    fh = logging.FileHandler(
        path / (dt.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + ".log"))
    sh = logging.StreamHandler(sys.stdout)
    formatter = logging.Formatter(
        "%(asctime)s %(name)s %(levelname)s %(message)s")

    fh.setFormatter(formatter)
    sh.setFormatter(formatter)
    logger.addHandler(fh)
    logger.addHandler(sh)
    return logger


def seed_torch(seed=1029):
    random.seed(seed)
    os.environ["PYTHONHASHSEED"] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    torch.backends.cudnn.deterministic = True
In [3]:
logger = get_logger(name="Main", tag="Pytorch-VGG16")

Data Loading

In [4]:
!ls ../input/imet-2019-fgvc6/
labels.csv  sample_submission.csv  test  train	train.csv
In [5]:
labels = pd.read_csv("../input/imet-2019-fgvc6/labels.csv")
train = pd.read_csv("../input/imet-2019-fgvc6/train.csv")
sample = pd.read_csv("../input/imet-2019-fgvc6/sample_submission.csv")
train.head()
Out[5]:
id attribute_ids
0 1000483014d91860 147 616 813
1 1000fe2e667721fe 51 616 734 813
2 1001614cb89646ee 776
3 10041eb49b297c08 51 671 698 813 1092
4 100501c227f8beea 13 404 492 903 1093
In [6]:
!cp ../input/pytorch-pretrained-image-models/* ./
!ls
__notebook__.ipynb  densenet121.pth  log	   resnet50.pth
__output__.json     densenet201.pth  resnet34.pth

DataLoader

In [7]:
# This loader is to extract 1024d features from the images.
class ImageDataLoader(data.DataLoader):
    def __init__(self, root_dir: Path, 
                 df: pd.DataFrame, 
                 mode="train", 
                 transforms=None):
        self._root = root_dir
        self.transform = transforms[mode]
        self._img_id = (df["id"] + ".png").values
        
    def __len__(self):
        return len(self._img_id)
    
    def __getitem__(self, idx):
        img_id = self._img_id[idx]
        file_name = self._root / img_id
        img = Image.open(file_name)
        
        if self.transform:
            img = self.transform(img)
            
        return [img]
    
    
data_transforms = {
    'train': vision.transforms.Compose([
        vision.transforms.RandomResizedCrop(224),
        vision.transforms.RandomHorizontalFlip(),
        vision.transforms.ToTensor(),
        vision.transforms.Normalize(
            [0.485, 0.456, 0.406], 
            [0.229, 0.224, 0.225])
    ]),
    'val': vision.transforms.Compose([
        vision.transforms.Resize(256),
        vision.transforms.CenterCrop(224),
        vision.transforms.ToTensor(),
        vision.transforms.Normalize(
            [0.485, 0.456, 0.406], 
            [0.229, 0.224, 0.225])
    ]),
}

data_transforms["test"] = data_transforms["val"]
In [8]:
# This loader is to be used for serving image tensors for the MLP.
class IMetDataset(data.Dataset):
    def __init__(self, tensor, device="cuda:0", labels=None):
        self.tensor = tensor
        self.labels = labels
        self.device= device
        
    def __len__(self):
        return self.tensor.size(0)
    
    def __getitem__(self, idx):
        tensor = self.tensor[idx, :]
        if self.labels is not None:
            label = self.labels[idx]
            label_tensor = torch.zeros((1, 1103))
            for i in label:
                label_tensor[0, int(i)] = 1
            label_tensor = label_tensor.to(self.device)
            return [tensor, label_tensor]
        else:
            return [tensor]

Model

Since passing images through the Densenet121 takes a lot of time, I don't want to take that time per epoch.

I will calculate the output of the Densenet121 beforehand so that converting raw images to processed feature vectors happens once.

In [9]:
class Classifier(nn.Module):
    def __init__(self):
        super(Classifier, self).__init__()
        
    def forward(self, x):
        return x


class Densenet121(nn.Module):
    def __init__(self, pretrained: Path):
        super(Densenet121, self).__init__()
        self.densenet121 = vision.models.densenet121()
        self.densenet121.load_state_dict(torch.load(pretrained))
        self.densenet121.classifier = Classifier()
        
        dense = nn.Sequential(*list(self.densenet121.children())[:-1])
        for param in dense.parameters():
            param.requires_grad = False
        
    def forward(self, x):
        return self.densenet121(x)
    
    
class MultiLayerPerceptron(nn.Module):
    def __init__(self):
        super(MultiLayerPerceptron, self).__init__()
        self.linear1 = nn.Linear(1024, 1024)
        self.relu = nn.ReLU()
        self.linear2 = nn.Linear(1024, 1103)
        self.dropout = nn.Dropout(0.5)
        self.sigmoid = nn.Sigmoid()
        
    def forward(self, x):
        x = self.relu(self.linear1(x))
        x = self.dropout(x)
        return self.sigmoid(self.linear2(x))

Data Preprocessing with Densenet121

In [10]:
train_dataset = ImageDataLoader(
    root_dir=Path("../input/imet-2019-fgvc6/train/"),
    df=train,
    mode="train",
    transforms=data_transforms)
train_loader = data.DataLoader(dataset=train_dataset,
                               shuffle=False,
                               batch_size=128)
test_dataset = ImageDataLoader(
    root_dir=Path("../input/imet-2019-fgvc6/test/"),
    df=sample,
    mode="test",
    transforms=data_transforms)
test_loader = data.DataLoader(dataset=test_dataset,
                              shuffle=False,
                              batch_size=128)
In [11]:
def get_feature_vector(df, loader, device):
    matrix = torch.zeros((df.shape[0], 1024)).to(device)
    model = Densenet121("densenet121.pth")
    model.to(device)
    batch = loader.batch_size
    for i, (i_batch,) in tqdm(enumerate(loader)):
        i_batch = i_batch.to(device)
        pred = model(i_batch).detach()
        matrix[i * batch:(i + 1) * batch] = pred
    return matrix
In [12]:
train_tensor = get_feature_vector(train, train_loader, "cuda:0")
test_tensor = get_feature_vector(sample, test_loader, "cuda:0")
854it [16:38,  1.05it/s]
59it [01:17,  1.04s/it]
In [13]:
del train_dataset, train_loader
del test_dataset, test_loader
gc.collect()
Out[13]:
875

Train Utilities

In [14]:
class Trainer:
    def __init__(self, 
                 model, 
                 logger,
                 n_splits=5,
                 seed=42,
                 device="cuda:0",
                 train_batch=32,
                 valid_batch=128,
                 kwargs={}):
        self.model = model
        self.logger = logger
        self.device = device
        self.n_splits = n_splits
        self.seed = seed
        self.train_batch = train_batch
        self.valid_batch = valid_batch
        self.kwargs = kwargs
        
        self.best_score = None
        self.tag = dt.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
        self.loss_fn = nn.BCELoss(reduction="mean").to(self.device)
        
        path = Path(f"bin/{self.tag}")
        path.mkdir(exist_ok=True, parents=True)
        self.path = path
        
    def fit(self, X, y, n_epochs=10):
        train_preds = np.zeros((len(X), 1103))
        fold = KFold(n_splits=self.n_splits, random_state=self.seed)
        for i, (trn_idx, val_idx) in enumerate(fold.split(X)):
            self.fold_num = i
            self.logger.info(f"Fold {i + 1}")
            X_train, X_val = X[trn_idx, :], X[val_idx, :]
            y_train, y_val = y[trn_idx], y[val_idx]
            
            valid_preds = self._fit(X_train, y_train, X_val, y_val, n_epochs)
            train_preds[val_idx] = valid_preds
        return train_preds
    
    def _fit(self, X_train, y_train, X_val, y_val, n_epochs):
        seed_torch(self.seed)
        train_dataset = IMetDataset(X_train, labels=y_train, device=self.device)
        train_loader = data.DataLoader(train_dataset, 
                                       batch_size=self.train_batch,
                                       shuffle=True)

        valid_dataset = IMetDataset(X_val, labels=y_val, device=self.device)
        valid_loader = data.DataLoader(valid_dataset,
                                       batch_size=self.valid_batch,
                                       shuffle=False)
        
        model = self.model(**self.kwargs)
        model.to(self.device)
        
        optimizer = optim.Adam(params=model.parameters(), 
                                lr=0.0001)
        scheduler = CosineAnnealingLR(optimizer, T_max=n_epochs)
        best_score = np.inf
        mb = master_bar(range(n_epochs))
        for epoch in mb:
            model.train()
            avg_loss = 0.0
            for i_batch, y_batch in progress_bar(train_loader, parent=mb):
                y_pred = model(i_batch)
                loss = self.loss_fn(y_pred, y_batch)
                optimizer.zero_grad()
                loss.backward()
                optimizer.step()
                avg_loss += loss.item() / len(train_loader)
            valid_preds, avg_val_loss = self._val(valid_loader, model)
            scheduler.step()

            self.logger.info("=========================================")
            self.logger.info(f"Epoch {epoch + 1} / {n_epochs}")
            self.logger.info("=========================================")
            self.logger.info(f"avg_loss: {avg_loss:.8f}")
            self.logger.info(f"avg_val_loss: {avg_val_loss:.8f}")
            
            if best_score > avg_val_loss:
                torch.save(model.state_dict(),
                           self.path / f"best{self.fold_num}.pth")
                self.logger.info(f"Save model at Epoch {epoch + 1}")
                best_score = avg_val_loss
        model.load_state_dict(torch.load(self.path / f"best{self.fold_num}.pth"))
        valid_preds, avg_val_loss = self._val(valid_loader, model)
        self.logger.info(f"Best Validation Loss: {avg_val_loss:.8f}")
        return valid_preds
    
    def _val(self, loader, model):
        model.eval()
        valid_preds = np.zeros((len(loader.dataset), 1103))
        avg_val_loss = 0.0
        for i, (i_batch, y_batch) in enumerate(loader):
            with torch.no_grad():
                y_pred = model(i_batch).detach()
                avg_val_loss += self.loss_fn(y_pred, y_batch).item() / len(loader)
                valid_preds[i * self.valid_batch:(i + 1) * self.valid_batch] = \
                    y_pred.cpu().numpy()
        return valid_preds, avg_val_loss
    
    def predict(self, X):
        dataset = IMetDataset(X, labels=None)
        loader = data.DataLoader(dataset, 
                                 batch_size=self.valid_batch, 
                                 shuffle=False)
        model = self.model(**self.kwargs)
        preds = np.zeros((X.size(0), 1103))
        for path in self.path.iterdir():
            with timer(f"Using {str(path)}", self.logger):
                model.load_state_dict(torch.load(path))
                model.to(self.device)
                model.eval()
                temp = np.zeros_like(preds)
                for i, (i_batch, ) in enumerate(loader):
                    with torch.no_grad():
                        y_pred = model(i_batch).detach()
                        temp[i * self.valid_batch:(i + 1) * self.valid_batch] = \
                            y_pred.cpu().numpy()
                preds += temp / self.n_splits
        return preds

Training

In [15]:
trainer = Trainer(MultiLayerPerceptron, logger, train_batch=64, kwargs={})
In [16]:
y = train.attribute_ids.map(lambda x: x.split()).values
valid_preds = trainer.fit(train_tensor, y, n_epochs=40)
2019-04-02 22:07:50,478 Main INFO Fold 1
Total time: 05:50

/opt/conda/lib/python3.6/site-packages/torch/nn/functional.py:2016: UserWarning: Using a target size (torch.Size([64, 1, 1103])) that is different to the input size (torch.Size([64, 1103])) is deprecated. Please ensure they have the same size.
  "Please ensure they have the same size.".format(target.size(), input.size()))
/opt/conda/lib/python3.6/site-packages/torch/nn/functional.py:2016: UserWarning: Using a target size (torch.Size([29, 1, 1103])) that is different to the input size (torch.Size([29, 1103])) is deprecated. Please ensure they have the same size.
  "Please ensure they have the same size.".format(target.size(), input.size()))
/opt/conda/lib/python3.6/site-packages/torch/nn/functional.py:2016: UserWarning: Using a target size (torch.Size([128, 1, 1103])) that is different to the input size (torch.Size([128, 1103])) is deprecated. Please ensure they have the same size.
  "Please ensure they have the same size.".format(target.size(), input.size()))
2019-04-02 22:07:59,766 Main INFO =========================================
2019-04-02 22:07:59,767 Main INFO Epoch 1 / 40
2019-04-02 22:07:59,768 Main INFO =========================================
2019-04-02 22:07:59,769 Main INFO avg_loss: 0.02454212
2019-04-02 22:07:59,769 Main INFO avg_val_loss: 0.01283460
2019-04-02 22:07:59,808 Main INFO Save model at Epoch 1
/opt/conda/lib/python3.6/site-packages/torch/nn/functional.py:2016: UserWarning: Using a target size (torch.Size([88, 1, 1103])) that is different to the input size (torch.Size([88, 1103])) is deprecated. Please ensure they have the same size.
  "Please ensure they have the same size.".format(target.size(), input.size()))
2019-04-02 22:08:09,122 Main INFO =========================================
2019-04-02 22:08:09,124 Main INFO Epoch 2 / 40
2019-04-02 22:08:09,125 Main INFO =========================================
2019-04-02 22:08:09,125 Main INFO avg_loss: 0.01284650
2019-04-02 22:08:09,126 Main INFO avg_val_loss: 0.01181298
2019-04-02 22:08:09,139 Main INFO Save model at Epoch 2
2019-04-02 22:08:17,623 Main INFO =========================================
2019-04-02 22:08:17,624 Main INFO Epoch 3 / 40
2019-04-02 22:08:17,624 Main INFO =========================================
2019-04-02 22:08:17,625 Main INFO avg_loss: 0.01197566
2019-04-02 22:08:17,626 Main INFO avg_val_loss: 0.01131342
2019-04-02 22:08:17,639 Main INFO Save model at Epoch 3
2019-04-02 22:08:26,158 Main INFO =========================================
2019-04-02 22:08:26,160 Main INFO Epoch 4 / 40
2019-04-02 22:08:26,160 Main INFO =========================================
2019-04-02 22:08:26,161 Main INFO avg_loss: 0.01146914
2019-04-02 22:08:26,162 Main INFO avg_val_loss: 0.01098631
2019-04-02 22:08:26,175 Main INFO Save model at Epoch 4
2019-04-02 22:08:36,112 Main INFO =========================================
2019-04-02 22:08:36,113 Main INFO Epoch 5 / 40
2019-04-02 22:08:36,114 Main INFO =========================================
2019-04-02 22:08:36,115 Main INFO avg_loss: 0.01112957
2019-04-02 22:08:36,116 Main INFO avg_val_loss: 0.01076920
2019-04-02 22:08:36,130 Main INFO Save model at Epoch 5
2019-04-02 22:08:44,939 Main INFO =========================================
2019-04-02 22:08:44,941 Main INFO Epoch 6 / 40
2019-04-02 22:08:44,941 Main INFO =========================================
2019-04-02 22:08:44,942 Main INFO avg_loss: 0.01087121
2019-04-02 22:08:44,942 Main INFO avg_val_loss: 0.01062898
2019-04-02 22:08:44,956 Main INFO Save model at Epoch 6
2019-04-02 22:08:53,514 Main INFO =========================================
2019-04-02 22:08:53,515 Main INFO Epoch 7 / 40
2019-04-02 22:08:53,516 Main INFO =========================================
2019-04-02 22:08:53,517 Main INFO avg_loss: 0.01066386
2019-04-02 22:08:53,518 Main INFO avg_val_loss: 0.01049846
2019-04-02 22:08:53,531 Main INFO Save model at Epoch 7
2019-04-02 22:09:02,026 Main INFO =========================================
2019-04-02 22:09:02,027 Main INFO Epoch 8 / 40
2019-04-02 22:09:02,027 Main INFO =========================================
2019-04-02 22:09:02,028 Main INFO avg_loss: 0.01049456
2019-04-02 22:09:02,029 Main INFO avg_val_loss: 0.01042114
2019-04-02 22:09:02,042 Main INFO Save model at Epoch 8
2019-04-02 22:09:10,553 Main INFO =========================================
2019-04-02 22:09:10,554 Main INFO Epoch 9 / 40
2019-04-02 22:09:10,555 Main INFO =========================================
2019-04-02 22:09:10,555 Main INFO avg_loss: 0.01035364
2019-04-02 22:09:10,556 Main INFO avg_val_loss: 0.01032429
2019-04-02 22:09:10,569 Main INFO Save model at Epoch 9
2019-04-02 22:09:19,486 Main INFO =========================================
2019-04-02 22:09:19,487 Main INFO Epoch 10 / 40
2019-04-02 22:09:19,488 Main INFO =========================================
2019-04-02 22:09:19,489 Main INFO avg_loss: 0.01022428
2019-04-02 22:09:19,489 Main INFO avg_val_loss: 0.01027195
2019-04-02 22:09:19,504 Main INFO Save model at Epoch 10
2019-04-02 22:09:28,926 Main INFO =========================================
2019-04-02 22:09:28,927 Main INFO Epoch 11 / 40
2019-04-02 22:09:28,927 Main INFO =========================================
2019-04-02 22:09:28,928 Main INFO avg_loss: 0.01011914
2019-04-02 22:09:28,928 Main INFO avg_val_loss: 0.01022184
2019-04-02 22:09:28,942 Main INFO Save model at Epoch 11
2019-04-02 22:09:37,460 Main INFO =========================================
2019-04-02 22:09:37,462 Main INFO Epoch 12 / 40
2019-04-02 22:09:37,462 Main INFO =========================================
2019-04-02 22:09:37,463 Main INFO avg_loss: 0.01001514
2019-04-02 22:09:37,463 Main INFO avg_val_loss: 0.01019556
2019-04-02 22:09:37,477 Main INFO Save model at Epoch 12
2019-04-02 22:09:46,004 Main INFO =========================================
2019-04-02 22:09:46,005 Main INFO Epoch 13 / 40
2019-04-02 22:09:46,006 Main INFO =========================================
2019-04-02 22:09:46,007 Main INFO avg_loss: 0.00992227
2019-04-02 22:09:46,007 Main INFO avg_val_loss: 0.01013475
2019-04-02 22:09:46,021 Main INFO Save model at Epoch 13
2019-04-02 22:09:54,597 Main INFO =========================================
2019-04-02 22:09:54,598 Main INFO Epoch 14 / 40
2019-04-02 22:09:54,599 Main INFO =========================================
2019-04-02 22:09:54,600 Main INFO avg_loss: 0.00984830
2019-04-02 22:09:54,600 Main INFO avg_val_loss: 0.01011313
2019-04-02 22:09:54,614 Main INFO Save model at Epoch 14
2019-04-02 22:10:03,115 Main INFO =========================================
2019-04-02 22:10:03,116 Main INFO Epoch 15 / 40
2019-04-02 22:10:03,117 Main INFO =========================================
2019-04-02 22:10:03,118 Main INFO avg_loss: 0.00976879
2019-04-02 22:10:03,118 Main INFO avg_val_loss: 0.01008825
2019-04-02 22:10:03,131 Main INFO Save model at Epoch 15
2019-04-02 22:10:11,619 Main INFO =========================================
2019-04-02 22:10:11,620 Main INFO Epoch 16 / 40
2019-04-02 22:10:11,621 Main INFO =========================================
2019-04-02 22:10:11,622 Main INFO avg_loss: 0.00970361
2019-04-02 22:10:11,622 Main INFO avg_val_loss: 0.01008143
2019-04-02 22:10:11,635 Main INFO Save model at Epoch 16
2019-04-02 22:10:20,155 Main INFO =========================================
2019-04-02 22:10:20,156 Main INFO Epoch 17 / 40
2019-04-02 22:10:20,157 Main INFO =========================================
2019-04-02 22:10:20,157 Main INFO avg_loss: 0.00963809
2019-04-02 22:10:20,158 Main INFO avg_val_loss: 0.01002651
2019-04-02 22:10:20,172 Main INFO Save model at Epoch 17
2019-04-02 22:10:28,827 Main INFO =========================================
2019-04-02 22:10:28,828 Main INFO Epoch 18 / 40
2019-04-02 22:10:28,829 Main INFO =========================================
2019-04-02 22:10:28,829 Main INFO avg_loss: 0.00958160
2019-04-02 22:10:28,830 Main INFO avg_val_loss: 0.01001838
2019-04-02 22:10:28,843 Main INFO Save model at Epoch 18
2019-04-02 22:10:37,536 Main INFO =========================================
2019-04-02 22:10:37,537 Main INFO Epoch 19 / 40
2019-04-02 22:10:37,538 Main INFO =========================================
2019-04-02 22:10:37,538 Main INFO avg_loss: 0.00952669
2019-04-02 22:10:37,539 Main INFO avg_val_loss: 0.01002354
2019-04-02 22:10:47,281 Main INFO =========================================
2019-04-02 22:10:47,282 Main INFO Epoch 20 / 40
2019-04-02 22:10:47,282 Main INFO =========================================
2019-04-02 22:10:47,283 Main INFO avg_loss: 0.00946760
2019-04-02 22:10:47,283 Main INFO avg_val_loss: 0.01000968
2019-04-02 22:10:47,297 Main INFO Save model at Epoch 20
2019-04-02 22:10:55,911 Main INFO =========================================
2019-04-02 22:10:55,912 Main INFO Epoch 21 / 40
2019-04-02 22:10:55,913 Main INFO =========================================
2019-04-02 22:10:55,913 Main INFO avg_loss: 0.00942477
2019-04-02 22:10:55,914 Main INFO avg_val_loss: 0.00998370
2019-04-02 22:10:55,927 Main INFO Save model at Epoch 21
2019-04-02 22:11:04,480 Main INFO =========================================
2019-04-02 22:11:04,481 Main INFO Epoch 22 / 40
2019-04-02 22:11:04,481 Main INFO =========================================
2019-04-02 22:11:04,482 Main INFO avg_loss: 0.00937976
2019-04-02 22:11:04,483 Main INFO avg_val_loss: 0.00997747
2019-04-02 22:11:04,496 Main INFO Save model at Epoch 22
2019-04-02 22:11:13,107 Main INFO =========================================
2019-04-02 22:11:13,109 Main INFO Epoch 23 / 40
2019-04-02 22:11:13,109 Main INFO =========================================
2019-04-02 22:11:13,110 Main INFO avg_loss: 0.00933588
2019-04-02 22:11:13,110 Main INFO avg_val_loss: 0.00996212
2019-04-02 22:11:13,124 Main INFO Save model at Epoch 23
2019-04-02 22:11:21,737 Main INFO =========================================
2019-04-02 22:11:21,739 Main INFO Epoch 24 / 40
2019-04-02 22:11:21,739 Main INFO =========================================
2019-04-02 22:11:21,740 Main INFO avg_loss: 0.00930078
2019-04-02 22:11:21,741 Main INFO avg_val_loss: 0.00996857
2019-04-02 22:11:30,280 Main INFO =========================================
2019-04-02 22:11:30,281 Main INFO Epoch 25 / 40
2019-04-02 22:11:30,282 Main INFO =========================================
2019-04-02 22:11:30,282 Main INFO avg_loss: 0.00926875
2019-04-02 22:11:30,283 Main INFO avg_val_loss: 0.00995972
2019-04-02 22:11:30,296 Main INFO Save model at Epoch 25
2019-04-02 22:11:38,836 Main INFO =========================================
2019-04-02 22:11:38,837 Main INFO Epoch 26 / 40
2019-04-02 22:11:38,838 Main INFO =========================================
2019-04-02 22:11:38,838 Main INFO avg_loss: 0.00923400
2019-04-02 22:11:38,839 Main INFO avg_val_loss: 0.00993973
2019-04-02 22:11:38,852 Main INFO Save model at Epoch 26
2019-04-02 22:11:47,378 Main INFO =========================================
2019-04-02 22:11:47,379 Main INFO Epoch 27 / 40
2019-04-02 22:11:47,380 Main INFO =========================================
2019-04-02 22:11:47,380 Main INFO avg_loss: 0.00920465
2019-04-02 22:11:47,381 Main INFO avg_val_loss: 0.00994709
2019-04-02 22:11:55,973 Main INFO =========================================
2019-04-02 22:11:55,974 Main INFO Epoch 28 / 40
2019-04-02 22:11:55,975 Main INFO =========================================
2019-04-02 22:11:55,976 Main INFO avg_loss: 0.00918702
2019-04-02 22:11:55,976 Main INFO avg_val_loss: 0.00993930
2019-04-02 22:11:55,989 Main INFO Save model at Epoch 28
2019-04-02 22:12:05,498 Main INFO =========================================
2019-04-02 22:12:05,500 Main INFO Epoch 29 / 40
2019-04-02 22:12:05,501 Main INFO =========================================
2019-04-02 22:12:05,502 Main INFO avg_loss: 0.00916385
2019-04-02 22:12:05,502 Main INFO avg_val_loss: 0.00992675
2019-04-02 22:12:05,517 Main INFO Save model at Epoch 29
2019-04-02 22:12:14,349 Main INFO =========================================
2019-04-02 22:12:14,350 Main INFO Epoch 30 / 40
2019-04-02 22:12:14,351 Main INFO =========================================
2019-04-02 22:12:14,351 Main INFO avg_loss: 0.00913773
2019-04-02 22:12:14,352 Main INFO avg_val_loss: 0.00992108
2019-04-02 22:12:14,365 Main INFO Save model at Epoch 30
2019-04-02 22:12:22,908 Main INFO =========================================
2019-04-02 22:12:22,909 Main INFO Epoch 31 / 40
2019-04-02 22:12:22,909 Main INFO =========================================
2019-04-02 22:12:22,910 Main INFO avg_loss: 0.00912388
2019-04-02 22:12:22,911 Main INFO avg_val_loss: 0.00994109
2019-04-02 22:12:31,418 Main INFO =========================================
2019-04-02 22:12:31,419 Main INFO Epoch 32 / 40
2019-04-02 22:12:31,420 Main INFO =========================================
2019-04-02 22:12:31,421 Main INFO avg_loss: 0.00910422
2019-04-02 22:12:31,421 Main INFO avg_val_loss: 0.00992830
2019-04-02 22:12:39,935 Main INFO =========================================
2019-04-02 22:12:39,936 Main INFO Epoch 33 / 40
2019-04-02 22:12:39,937 Main INFO =========================================
2019-04-02 22:12:39,937 Main INFO avg_loss: 0.00909780
2019-04-02 22:12:39,938 Main INFO avg_val_loss: 0.00992444
2019-04-02 22:12:48,447 Main INFO =========================================
2019-04-02 22:12:48,448 Main INFO Epoch 34 / 40
2019-04-02 22:12:48,449 Main INFO =========================================
2019-04-02 22:12:48,450 Main INFO avg_loss: 0.00908681
2019-04-02 22:12:48,450 Main INFO avg_val_loss: 0.00991563
2019-04-02 22:12:48,463 Main INFO Save model at Epoch 34
2019-04-02 22:12:57,062 Main INFO =========================================
2019-04-02 22:12:57,063 Main INFO Epoch 35 / 40
2019-04-02 22:12:57,063 Main INFO =========================================
2019-04-02 22:12:57,064 Main INFO avg_loss: 0.00907445
2019-04-02 22:12:57,065 Main INFO avg_val_loss: 0.00992488
2019-04-02 22:13:05,550 Main INFO =========================================
2019-04-02 22:13:05,551 Main INFO Epoch 36 / 40
2019-04-02 22:13:05,552 Main INFO =========================================
2019-04-02 22:13:05,552 Main INFO avg_loss: 0.00906618
2019-04-02 22:13:05,553 Main INFO avg_val_loss: 0.00992139
2019-04-02 22:13:14,046 Main INFO =========================================
2019-04-02 22:13:14,047 Main INFO Epoch 37 / 40
2019-04-02 22:13:14,048 Main INFO =========================================
2019-04-02 22:13:14,048 Main INFO avg_loss: 0.00905837
2019-04-02 22:13:14,049 Main INFO avg_val_loss: 0.00991973
2019-04-02 22:13:23,189 Main INFO =========================================
2019-04-02 22:13:23,190 Main INFO Epoch 38 / 40
2019-04-02 22:13:23,191 Main INFO =========================================
2019-04-02 22:13:23,191 Main INFO avg_loss: 0.00904998
2019-04-02 22:13:23,192 Main INFO avg_val_loss: 0.00991816
2019-04-02 22:13:32,352 Main INFO =========================================
2019-04-02 22:13:32,353 Main INFO Epoch 39 / 40
2019-04-02 22:13:32,354 Main INFO =========================================
2019-04-02 22:13:32,355 Main INFO avg_loss: 0.00905254
2019-04-02 22:13:32,355 Main INFO avg_val_loss: 0.00992003
2019-04-02 22:13:41,507 Main INFO =========================================
2019-04-02 22:13:41,508 Main INFO Epoch 40 / 40
2019-04-02 22:13:41,509 Main INFO =========================================
2019-04-02 22:13:41,510 Main INFO avg_loss: 0.00905516
2019-04-02 22:13:41,511 Main INFO avg_val_loss: 0.00992060
2019-04-02 22:13:43,309 Main INFO Best Validation Loss: 0.00991563
2019-04-02 22:13:43,472 Main INFO Fold 2
60.00% [24/40 03:29<02:19]
84.33% [1152/1366 00:05<00:01]
2019-04-02 22:13:52,690 Main INFO =========================================
2019-04-02 22:13:52,692 Main INFO Epoch 1 / 40
2019-04-02 22:13:52,694 Main INFO =========================================
2019-04-02 22:13:52,697 Main INFO avg_loss: 0.02451130
2019-04-02 22:13:52,698 Main INFO avg_val_loss: 0.01296820
2019-04-02 22:13:52,710 Main INFO Save model at Epoch 1
2019-04-02 22:14:01,232 Main INFO =========================================
2019-04-02 22:14:01,233 Main INFO Epoch 2 / 40
2019-04-02 22:14:01,234 Main INFO =========================================
2019-04-02 22:14:01,234 Main INFO avg_loss: 0.01281673
2019-04-02 22:14:01,235 Main INFO avg_val_loss: 0.01191164
2019-04-02 22:14:01,248 Main INFO Save model at Epoch 2
2019-04-02 22:14:09,783 Main INFO =========================================
2019-04-02 22:14:09,785 Main INFO Epoch 3 / 40
2019-04-02 22:14:09,786 Main INFO =========================================
2019-04-02 22:14:09,786 Main INFO avg_loss: 0.01193949
2019-04-02 22:14:09,787 Main INFO avg_val_loss: 0.01140887
2019-04-02 22:14:09,800 Main INFO Save model at Epoch 3
2019-04-02 22:14:18,345 Main INFO =========================================
2019-04-02 22:14:18,346 Main INFO Epoch 4 / 40
2019-04-02 22:14:18,346 Main INFO =========================================
2019-04-02 22:14:18,347 Main INFO avg_loss: 0.01143907
2019-04-02 22:14:18,348 Main INFO avg_val_loss: 0.01109527
2019-04-02 22:14:18,361 Main INFO Save model at Epoch 4
2019-04-02 22:14:26,900 Main INFO =========================================
2019-04-02 22:14:26,901 Main INFO Epoch 5 / 40
2019-04-02 22:14:26,902 Main INFO =========================================
2019-04-02 22:14:26,903 Main INFO avg_loss: 0.01109554
2019-04-02 22:14:26,903 Main INFO avg_val_loss: 0.01087336
2019-04-02 22:14:26,916 Main INFO Save model at Epoch 5
2019-04-02 22:14:35,464 Main INFO =========================================
2019-04-02 22:14:35,465 Main INFO Epoch 6 / 40
2019-04-02 22:14:35,466 Main INFO =========================================
2019-04-02 22:14:35,466 Main INFO avg_loss: 0.01084370
2019-04-02 22:14:35,467 Main INFO avg_val_loss: 0.01074624
2019-04-02 22:14:35,480 Main INFO Save model at Epoch 6
2019-04-02 22:14:44,727 Main INFO =========================================
2019-04-02 22:14:44,728 Main INFO Epoch 7 / 40
2019-04-02 22:14:44,729 Main INFO =========================================
2019-04-02 22:14:44,730 Main INFO avg_loss: 0.01063413
2019-04-02 22:14:44,730 Main INFO avg_val_loss: 0.01060765
2019-04-02 22:14:44,745 Main INFO Save model at Epoch 7
2019-04-02 22:14:54,174 Main INFO =========================================
2019-04-02 22:14:54,175 Main INFO Epoch 8 / 40
2019-04-02 22:14:54,176 Main INFO =========================================
2019-04-02 22:14:54,176 Main INFO avg_loss: 0.01046244
2019-04-02 22:14:54,177 Main INFO avg_val_loss: 0.01052366
2019-04-02 22:14:54,190 Main INFO Save model at Epoch 8
2019-04-02 22:15:02,743 Main INFO =========================================
2019-04-02 22:15:02,744 Main INFO Epoch 9 / 40
2019-04-02 22:15:02,744 Main INFO =========================================
2019-04-02 22:15:02,745 Main INFO avg_loss: 0.01031934
2019-04-02 22:15:02,746 Main INFO avg_val_loss: 0.01044007
2019-04-02 22:15:02,759 Main INFO Save model at Epoch 9
2019-04-02 22:15:11,284 Main INFO =========================================
2019-04-02 22:15:11,285 Main INFO Epoch 10 / 40
2019-04-02 22:15:11,286 Main INFO =========================================
2019-04-02 22:15:11,286 Main INFO avg_loss: 0.01019201
2019-04-02 22:15:11,287 Main INFO avg_val_loss: 0.01039488
2019-04-02 22:15:11,300 Main INFO Save model at Epoch 10
2019-04-02 22:15:19,842 Main INFO =========================================
2019-04-02 22:15:19,843 Main INFO Epoch 11 / 40
2019-04-02 22:15:19,846 Main INFO =========================================
2019-04-02 22:15:19,847 Main INFO avg_loss: 0.01008467
2019-04-02 22:15:19,848 Main INFO avg_val_loss: 0.01035010
2019-04-02 22:15:19,862 Main INFO Save model at Epoch 11
2019-04-02 22:15:28,422 Main INFO =========================================
2019-04-02 22:15:28,423 Main INFO Epoch 12 / 40
2019-04-02 22:15:28,424 Main INFO =========================================
2019-04-02 22:15:28,425 Main INFO avg_loss: 0.00998682
2019-04-02 22:15:28,425 Main INFO avg_val_loss: 0.01030926
2019-04-02 22:15:28,439 Main INFO Save model at Epoch 12
2019-04-02 22:15:37,076 Main INFO =========================================
2019-04-02 22:15:37,077 Main INFO Epoch 13 / 40
2019-04-02 22:15:37,077 Main INFO =========================================
2019-04-02 22:15:37,078 Main INFO avg_loss: 0.00989225
2019-04-02 22:15:37,079 Main INFO avg_val_loss: 0.01023603
2019-04-02 22:15:37,092 Main INFO Save model at Epoch 13
2019-04-02 22:15:45,628 Main INFO =========================================
2019-04-02 22:15:45,629 Main INFO Epoch 14 / 40
2019-04-02 22:15:45,630 Main INFO =========================================
2019-04-02 22:15:45,630 Main INFO avg_loss: 0.00981475
2019-04-02 22:15:45,631 Main INFO avg_val_loss: 0.01022511
2019-04-02 22:15:45,644 Main INFO Save model at Epoch 14
2019-04-02 22:15:54,198 Main INFO =========================================
2019-04-02 22:15:54,199 Main INFO Epoch 15 / 40
2019-04-02 22:15:54,199 Main INFO =========================================
2019-04-02 22:15:54,200 Main INFO avg_loss: 0.00974034
2019-04-02 22:15:54,201 Main INFO avg_val_loss: 0.01020647
2019-04-02 22:15:54,214 Main INFO Save model at Epoch 15
2019-04-02 22:16:03,226 Main INFO =========================================
2019-04-02 22:16:03,227 Main INFO Epoch 16 / 40
2019-04-02 22:16:03,228 Main INFO =========================================
2019-04-02 22:16:03,228 Main INFO avg_loss: 0.00967017
2019-04-02 22:16:03,229 Main INFO avg_val_loss: 0.01020109
2019-04-02 22:16:03,244 Main INFO Save model at Epoch 16
2019-04-02 22:16:12,620 Main INFO =========================================
2019-04-02 22:16:12,621 Main INFO Epoch 17 / 40
2019-04-02 22:16:12,622 Main INFO =========================================
2019-04-02 22:16:12,622 Main INFO avg_loss: 0.00960502
2019-04-02 22:16:12,623 Main INFO avg_val_loss: 0.01014905
2019-04-02 22:16:12,636 Main INFO Save model at Epoch 17
2019-04-02 22:16:21,199 Main INFO =========================================
2019-04-02 22:16:21,200 Main INFO Epoch 18 / 40
2019-04-02 22:16:21,201 Main INFO =========================================
2019-04-02 22:16:21,202 Main INFO avg_loss: 0.00954806
2019-04-02 22:16:21,203 Main INFO avg_val_loss: 0.01014502
2019-04-02 22:16:21,216 Main INFO Save model at Epoch 18
2019-04-02 22:16:29,752 Main INFO =========================================
2019-04-02 22:16:29,753 Main INFO Epoch 19 / 40
2019-04-02 22:16:29,753 Main INFO =========================================
2019-04-02 22:16:29,754 Main INFO avg_loss: 0.00949257
2019-04-02 22:16:29,755 Main INFO avg_val_loss: 0.01013664
2019-04-02 22:16:29,769 Main INFO Save model at Epoch 19
2019-04-02 22:16:38,299 Main INFO =========================================
2019-04-02 22:16:38,300 Main INFO Epoch 20 / 40
2019-04-02 22:16:38,301 Main INFO =========================================
2019-04-02 22:16:38,302 Main INFO avg_loss: 0.00944125
2019-04-02 22:16:38,302 Main INFO avg_val_loss: 0.01011565
2019-04-02 22:16:38,315 Main INFO Save model at Epoch 20
2019-04-02 22:16:46,851 Main INFO =========================================
2019-04-02 22:16:46,852 Main INFO Epoch 21 / 40
2019-04-02 22:16:46,852 Main INFO =========================================
2019-04-02 22:16:46,853 Main INFO avg_loss: 0.00939168
2019-04-02 22:16:46,853 Main INFO avg_val_loss: 0.01010595
2019-04-02 22:16:46,867 Main INFO Save model at Epoch 21
2019-04-02 22:16:55,433 Main INFO =========================================
2019-04-02 22:16:55,434 Main INFO Epoch 22 / 40
2019-04-02 22:16:55,435 Main INFO =========================================
2019-04-02 22:16:55,436 Main INFO avg_loss: 0.00934240
2019-04-02 22:16:55,436 Main INFO avg_val_loss: 0.01008844
2019-04-02 22:16:55,449 Main INFO Save model at Epoch 22
2019-04-02 22:17:04,028 Main INFO =========================================
2019-04-02 22:17:04,029 Main INFO Epoch 23 / 40
2019-04-02 22:17:04,030 Main INFO =========================================
2019-04-02 22:17:04,031 Main INFO avg_loss: 0.00930924
2019-04-02 22:17:04,031 Main INFO avg_val_loss: 0.01007159
2019-04-02 22:17:04,045 Main INFO Save model at Epoch 23
2019-04-02 22:17:12,610 Main INFO =========================================
2019-04-02 22:17:12,611 Main INFO Epoch 24 / 40
2019-04-02 22:17:12,612 Main INFO =========================================
2019-04-02 22:17:12,613 Main INFO avg_loss: 0.00927431
2019-04-02 22:17:12,613 Main INFO avg_val_loss: 0.01008604

Post process - threshold search -

Since I used sigmoid for the activation, I've got the 1103 probability output for each data row.

I need to decide threshold for this.There are two ways to deal with this.

  • Class-wise threshold search
    • Takes some time but it's natural.
  • One threshold for all the class
    • Low cost way.

UPDATE I will use the first -> second one.

In [17]:
def threshold_search(y_pred, y_true):
    score = []
    candidates = np.arange(0, 1.0, 0.01)
    for th in progress_bar(candidates):
        yp = (y_pred > th).astype(int)
        score.append(fbeta_score(y_pred=yp, y_true=y_true, beta=2, average="samples"))
    score = np.array(score)
    pm = score.argmax()
    best_th, best_score = candidates[pm], score[pm]
    return best_th, best_score
In [18]:
y_true = np.zeros((train.shape[0], 1103)).astype(int)
for i, row in enumerate(y):
    for idx in row:
        y_true[i, int(idx)] = 1
In [19]:
best_threshold, best_score = threshold_search(valid_preds, y_true)
best_score
100.00% [100/100 16:05<00:00]
/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.
  'precision', 'predicted', average, warn_for)
Out[19]:
0.4485115751242815

Prediction for test data

In [20]:
test_preds = trainer.predict(test_tensor)
2019-04-02 22:53:21,066 Main INFO [Using bin/2019-04-02-22-07-50/best1.pth] done in 0.2601180076599121 s
2019-04-02 22:53:21,270 Main INFO [Using bin/2019-04-02-22-07-50/best3.pth] done in 0.2036607265472412 s
2019-04-02 22:53:21,470 Main INFO [Using bin/2019-04-02-22-07-50/best0.pth] done in 0.1983020305633545 s
2019-04-02 22:53:21,663 Main INFO [Using bin/2019-04-02-22-07-50/best2.pth] done in 0.1916201114654541 s
2019-04-02 22:53:21,860 Main INFO [Using bin/2019-04-02-22-07-50/best4.pth] done in 0.19613099098205566 s
In [21]:
preds = (test_preds > best_threshold).astype(int)
In [22]:
prediction = []
for i in range(preds.shape[0]):
    pred1 = np.argwhere(preds[i] == 1.0).reshape(-1).tolist()
    pred_str = " ".join(list(map(str, pred1)))
    prediction.append(pred_str)
    
sample.attribute_ids = prediction
sample.to_csv("submission.csv", index=False)
sample.head()
Out[22]:
id attribute_ids
0 10023b2cc4ed5f68 223 343 369 587 766 1059
1 100fbe75ed8fd887 93 231 369 489 1039
2 101b627524a04f19 79 482 497 498 728 784
3 10234480c41284c6 13 51 111 147 189 480 483 725 776 813 830 1046
4 1023b0e2636dcea8 147 189 194 322 477 489 584 612 671 780 813 95...