kernel266d1c7f1f

From: https://www.kaggle.com/itwice/kernel266d1c7f1f

Author: Nikita Shigarov

Score: 0.5

In [1]:
import gc
import os
import sys
import time
import random
import logging
import datetime as dt

import numpy as np
import pandas as pd

import torch
import torch.nn as nn
import torch.optim as optim
import torch.utils.data as data
import torch.nn.functional as F
import torchvision as vision

from torch.optim.lr_scheduler import CosineAnnealingLR

from pathlib import Path
from PIL import Image
from contextlib import contextmanager

from joblib import Parallel, delayed
from tqdm import tqdm
from fastprogress import master_bar, progress_bar

from sklearn.model_selection import KFold
from sklearn.metrics import fbeta_score
In [2]:
#!mkdir -p /tmp/.torch/models/
#!wget -O /tmp/.torch/models/se_resnet152-d17c99b7.pth http://data.lip6.fr/cadene/pretrainedmodels/se_resnet152-d17c99b7.pth
#import pretrainedmodels
torch.cuda.is_available()
Out[2]:
True
In [3]:
@contextmanager
def timer(name="Main", logger=None):
    t0 = time.time()
    yield
    msg = f"[{name}] done in {time.time() - t0} s"
    if logger is not None:
        logger.info(msg)
    else:
        print(msg)
        

def get_logger(name="Main", tag="exp", log_dir="log/"):
    log_path = Path(log_dir)
    path = log_path / tag
    path.mkdir(exist_ok=True, parents=True)

    logger = logging.getLogger(name)
    logger.setLevel(logging.INFO)

    fh = logging.FileHandler(
        path / (dt.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + ".log"))
    sh = logging.StreamHandler(sys.stdout)
    formatter = logging.Formatter(
        "%(asctime)s %(name)s %(levelname)s %(message)s")

    fh.setFormatter(formatter)
    sh.setFormatter(formatter)
    logger.addHandler(fh)
    logger.addHandler(sh)
    return logger


def seed_torch(seed=1029):
    random.seed(seed)
    os.environ["PYTHONHASHSEED"] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    torch.backends.cudnn.deterministic = True
In [4]:
logger = get_logger(name="Main", tag="Pytorch-VGG16")
In [5]:
@contextmanager
def timer(name="Main", logger=None):
    t0 = time.time()
    yield
    msg = f"[{name}] done in {time.time() - t0} s"
    if logger is not None:
        logger.info(msg)
    else:
        print(msg)
        

def get_logger(name="Main", tag="exp", log_dir="log/"):
    log_path = Path(log_dir)
    path = log_path / tag
    path.mkdir(exist_ok=True, parents=True)

    logger = logging.getLogger(name)
    logger.setLevel(logging.INFO)

    fh = logging.FileHandler(
        path / (dt.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + ".log"))
    sh = logging.StreamHandler(sys.stdout)
    formatter = logging.Formatter(
        "%(asctime)s %(name)s %(levelname)s %(message)s")

    fh.setFormatter(formatter)
    sh.setFormatter(formatter)
    logger.addHandler(fh)
    logger.addHandler(sh)
    return logger


def seed_torch(seed=1029):
    random.seed(seed)
    os.environ["PYTHONHASHSEED"] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    torch.backends.cudnn.deterministic = True
In [6]:
!ls ../input
labels = pd.read_csv("../input/imet-2019-fgvc6/labels.csv")
train = pd.read_csv("../input/imet-2019-fgvc6/train.csv")
sample = pd.read_csv("../input/imet-2019-fgvc6/sample_submission.csv")
train.head()

cultures = [x for x in labels.attribute_name.values if x.startswith("culture")]
tags = [x for x in labels.attribute_name.values if x.startswith("tag")]
len(cultures), len(tags)
imet-2019-fgvc6  pytorch-pretrained-image-models
Out[6]:
(398, 705)
In [7]:
import cv2
def split_culture_tag(x):
    cultures_ = list()
    tags_ = list()
    for i in x.split(" "):
        if int(i) <= len(cultures):
            cultures_.append(i)
        else:
            tags_.append(str(int(i) - len(cultures)))
    if not cultures_:
        cultures_.append(str(len(cultures)))
    if not tags_:
        tags_.append(str(len(tags)))
    return " ".join(cultures_), " ".join(tags_)

culture_ids = list()
tag_ids = list()

for v in tqdm(train.attribute_ids.values):
    c, t = split_culture_tag(v)
    culture_ids.append(c)
    tag_ids.append(t)

num_classes_c = len(cultures) + 1
num_classes_t = len(tags) + 1

train["culture_ids"] = culture_ids
train["tag_ids"] = tag_ids


def obtain_y_c(ids):
    y = np.zeros(num_classes_c)
    for idx in ids.split(" "):
        y[int(idx)] = 1
    return y

def obtain_y_t(ids):
    y = np.zeros(num_classes_t)
    for idx in ids.split(" "):
        y[int(idx)] = 1
    return y

paths = ["../input/imet-2019-fgvc6/train/{}.png".format(x) for x in train.id.values]

targets_c = np.array([obtain_y_c(y) for y in train.culture_ids.values])
targets_t = np.array([obtain_y_t(y) for y in train.tag_ids.values])
print(targets_c.shape)

def rem_bkg(img):
    y_size,x_size,col = img.shape
    
    for y in range(y_size):
        for r in range(1,6):
            col = img[y, x_size-r] 
            img[np.where((img == col).all(axis = 2))] = [255,255,255]
        for l in range(5):
            col = img[y, l] 
            img[np.where((img == col).all(axis = 2))] = [255,255,255]

    for x in range(x_size):
        for d in range(1,6):
            col = img[y_size-d, x] 
            img[np.where((img == col).all(axis = 2))] = [255,255,255]
        for u in range(5):
            col = img[u, x] 
            img[np.where((img == col).all(axis = 2))] = [255,255,255]
    
    return img

class ImageDataLoader(data.DataLoader):
    def __init__(self, root_dir: Path, 
                 df: pd.DataFrame, 
                 mode="train", 
                 transforms=None):
        self._root = root_dir
        self.transform = transforms[mode]
        self._img_id = (df["id"] + ".png").values
        
    def __len__(self):
        return len(self._img_id)
    
    def __getitem__(self, idx):
        img_id = self._img_id[idx]
        file_name = self._root / img_id
        img = Image.open(file_name)
        #img = cv2.imread(file_name.absolute().as_posix())[...,[2, 1, 0]]
        #img = rem_bkg(img)
        #img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
        #img = Image.fromarray(img)
        if self.transform:
            img = self.transform(img)
            
        return [img]
    
    
data_transforms = {
    'train': vision.transforms.Compose([
        vision.transforms.RandomResizedCrop(224),
        vision.transforms.RandomHorizontalFlip(),
        vision.transforms.ToTensor(),
        vision.transforms.Normalize(
            [0.485, 0.456, 0.406], 
            [0.229, 0.224, 0.225])
    ]),
    'val': vision.transforms.Compose([
        vision.transforms.Resize(256),
        vision.transforms.CenterCrop(224),
        vision.transforms.ToTensor(),
        vision.transforms.Normalize(
            [0.485, 0.456, 0.406], 
            [0.229, 0.224, 0.225])
    ]),
}

data_transforms["test"] = data_transforms["val"]
100%|██████████| 109237/109237 [00:00<00:00, 236246.27it/s]
(109237, 399)
In [8]:
class IMetDataset(data.Dataset):
    def __init__(self, tensor, device="cuda:0", labels=None):
        self.tensor = tensor
        self.labels = labels
        self.device= device
        
    def __len__(self):
        return self.tensor.size(0)
    
    def __getitem__(self, idx):
        tensor = self.tensor[idx, :]
        if self.labels is not None:
            label = self.labels[idx]
            label_tensor = torch.zeros((1, 1103))
            y_c = torch.FloatTensor(targets_c[idx]).to(self.device)
            y_t = torch.FloatTensor(targets_t[idx]).to(self.device)
            for i in label:
                label_tensor[0, int(i)] = 1
            label_tensor = label_tensor.to(self.device)
            return [tensor, [y_c, y_t]]
        else:
            return [tensor]
In [9]:
class Classifier(nn.Module):
    def __init__(self):
        super(Classifier, self).__init__()
        
    def forward(self, x):
        return x


class Densenet121(nn.Module):
    def __init__(self, pretrained: Path):
        super(Densenet121, self).__init__()
        self.densenet121 = vision.models.densenet121()
        self.densenet121.load_state_dict(torch.load(pretrained))
        self.densenet121.classifier = Classifier()
        
        dense = nn.Sequential(*list(self.densenet121.children())[:-1])
        for param in dense.parameters():
            param.requires_grad = False
        
    def forward(self, x):
        return self.densenet121(x)
    
class Resnet50(nn.Module):
    def __init__(self, pretrained: Path):
        super(Resnet50, self).__init__()
        self.resnet50 = vision.models.resnet50()
        self.resnet50.load_state_dict(torch.load(pretrained))
        self.resnet50.classifier = Classifier()
        
        dense = nn.Sequential(*list(self.resnet50.children())[:-1])
        for param in dense.parameters():
            param.requires_grad = False
        
    def forward(self, x):
        return self.resnet50(x)
    
    
class MultiLayerPerceptron(nn.Module):
    def __init__(self):
        super(MultiLayerPerceptron, self).__init__()
        self.linear1 = nn.Linear(1024, 1024)
        self.relu = nn.ReLU()
        self.linear11 = nn.Linear(1024, 1024)
        self.relu2 = nn.ReLU()
        self.linear2 = nn.Linear(1024, 1103)
        self.dropout = nn.Dropout(0.5)
        self.sigmoid = nn.Sigmoid()
        
    def forward(self, x):
        x = self.relu(self.linear1(x))
        x = self.relu2(self.linear11(x))
        x = self.dropout(x)
        return self.sigmoid(self.linear2(x))
    
class MultiLayerPerceptron1(nn.Module):
    def __init__(self):
        super(MultiLayerPerceptron1, self).__init__()
        self.linear1 = nn.Linear(1024, 1024)
        self.relu = nn.ReLU()
        self.linear11 = nn.Linear(1024, 1024)
        self.relu2 = nn.ReLU()
        self.linear2 = nn.Linear(1024, 399)
        self.dropout = nn.Dropout(0.5)
        self.sigmoid = nn.Sigmoid()
        
    def forward(self, x):
        x = self.relu(self.linear1(x))
        x = self.relu2(self.linear11(x))
        x = self.dropout(x)
        return self.sigmoid(self.linear2(x))
    
class MultiLayerPerceptron2(nn.Module):
    def __init__(self):
        super(MultiLayerPerceptron2, self).__init__()
        self.linear1 = nn.Linear(1024, 1024)
        self.relu = nn.ReLU()
        self.linear11 = nn.Linear(1024, 1024)
        self.relu2 = nn.ReLU()
        self.linear2 = nn.Linear(1024, 706)
        self.dropout = nn.Dropout(0.5)
        self.sigmoid = nn.Sigmoid()
        
    def forward(self, x):
        x = self.relu(self.linear1(x))
        x = self.relu2(self.linear11(x))
        x = self.dropout(x)
        return self.sigmoid(self.linear2(x))
In [10]:
train_dataset = ImageDataLoader(
    root_dir=Path("../input/imet-2019-fgvc6/train/"),
    df=train,
    mode="train",
    transforms=data_transforms)
train_loader = data.DataLoader(dataset=train_dataset,
                               shuffle=False,
                               batch_size=64)
test_dataset = ImageDataLoader(
    root_dir=Path("../input/imet-2019-fgvc6/test/"),
    df=sample,
    mode="test",
    transforms=data_transforms)
test_loader = data.DataLoader(dataset=test_dataset,
                              shuffle=False,
                              batch_size=64)
In [11]:
from torchvision import models
def get_feature_vector(df, loader, device):
    matrix = torch.zeros((df.shape[0], 1024)).to(device)
    model = Densenet121('../input/pytorch-pretrained-image-models/densenet121.pth') #Resnet50('../input/pytorch-pretrained-image-models/resnet50.pth')

    model.to(device)
    batch = loader.batch_size
    for i, (i_batch,) in tqdm(enumerate(loader)):
        i_batch = i_batch.to(device)
        pred = model(i_batch).detach()
        matrix[i * batch:(i + 1) * batch] = pred
    return matrix
In [12]:
train_tensor = get_feature_vector(train, train_loader, "cuda:0")
test_tensor = get_feature_vector(sample, test_loader, "cuda:0")
1707it [19:34,  1.50it/s]
117it [01:25,  1.72it/s]
In [13]:
del train_dataset, train_loader
del test_dataset, test_loader
gc.collect()
Out[13]:
763
In [14]:
from numpy.random import beta
class FocalLoss(nn.Module):
    def __init__(self, alpha=0.5, gamma=2, logits=False, reduce=True):
        super(FocalLoss, self).__init__()
        self.alpha = alpha
        self.gamma = gamma
        self.logits = logits
        self.reduce = reduce

    def forward(self, inputs, targets):
        if self.logits:
            BCE_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduce=False)
        else:
            BCE_loss = F.binary_cross_entropy(inputs, targets, reduce=False)
        pt = torch.exp(-BCE_loss)
        F_loss = self.alpha * (1-pt)**self.gamma * BCE_loss

        if self.reduce:
            return torch.sum(F_loss)
        else:
            return F_loss
        
def mixup(input, target, gamma):
    # target is onehot format!
    perm = torch.randperm(input.size(0))
    perm_input = input[perm]
    perm_target = target[perm]
    return input.mul_(gamma).add_(1 - gamma, perm_input), target.mul_(gamma).add_(1 - gamma, perm_target)

    return mixed_x, mixed_y

class Trainer:
    def __init__(self, 
                 model1,
                 model2,
                 logger,
                 n_splits=5,
                 seed=42,
                 device="cuda:0",
                 train_batch=32,
                 valid_batch=128,
                 kwargs={}):
        self.model1 = model1
        self.model2 = model2
        self.logger = logger
        self.device = device
        self.n_splits = n_splits
        self.seed = seed
        self.train_batch = train_batch
        self.valid_batch = valid_batch
        self.kwargs = kwargs
        
        self.best_score = None
        self.tag = dt.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
        self.loss_fn = nn.BCELoss(reduction="mean").to(self.device)
        
        path = Path(f"bin1/{self.tag}")
        path.mkdir(exist_ok=True, parents=True)
        self.path1 = path
        path = Path(f"bin2/{self.tag}")
        path.mkdir(exist_ok=True, parents=True)
        self.path2 = path
        
    def fit(self, X, y, n_epochs=10):
        train_preds1 = np.zeros((len(X), num_classes_c))
        train_preds2 = np.zeros((len(X), num_classes_t))
        fold = KFold(n_splits=self.n_splits, random_state=self.seed)
        for i, (trn_idx, val_idx) in enumerate(fold.split(X)):
            self.fold_num = i
            self.logger.info(f"Fold {i + 1}")
            X_train, X_val = X[trn_idx, :], X[val_idx, :]
            y_train, y_val = y[trn_idx], y[val_idx]
            
            valid_preds1, valid_preds2 = self._fit(X_train, y_train, X_val, y_val, n_epochs)
            #print('tp1 ' + str(train_preds1.shape[1]))
            #print('vp1 ' + str(valid_preds1.shape[1]))
            train_preds1[val_idx] = valid_preds1
            train_preds2[val_idx] = valid_preds2
        return train_preds1, train_preds2
    
    def _fit(self, X_train, y_train, X_val, y_val, n_epochs):
        seed_torch(self.seed)
        train_dataset = IMetDataset(X_train, labels=y_train, device=self.device)
        train_loader = data.DataLoader(train_dataset, 
                                       batch_size=self.train_batch,
                                       shuffle=True)

        valid_dataset = IMetDataset(X_val, labels=y_val, device=self.device)
        valid_loader = data.DataLoader(valid_dataset,
                                       batch_size=self.valid_batch,
                                       shuffle=False)
        
        model1 = self.model1(**self.kwargs)
        model1.to(self.device)
        
        model2 = self.model2(**self.kwargs)
        model2.to(self.device)
        
        optimizer1 = optim.Adam(params=model1.parameters(), 
                                lr=0.0001)
        optimizer2 = optim.Adam(params=model2.parameters(), 
                                lr=0.0001)
        scheduler1 = CosineAnnealingLR(optimizer1, T_max=n_epochs)
        scheduler2 = CosineAnnealingLR(optimizer2, T_max=n_epochs)
        best_score1 = np.inf
        best_score2 = np.inf
        mb = master_bar(range(n_epochs))
        for epoch in mb:
            model1.train()
            model2.train()
            avg_loss1 = 0.0
            avg_loss2 = 0.0
            for i_batch, y_batch in progress_bar(train_loader, parent=mb):
                #i_batch, y_batch = mixup(i_batch, y_batch, beta(1.0, 1.0))
                y_pred1 = model1(i_batch)
                y_pred2 = model2(i_batch)
                loss1 = self.loss_fn(y_pred1, y_batch[0])
                loss2 = self.loss_fn(y_pred2, y_batch[1])
                optimizer1.zero_grad()
                optimizer2.zero_grad()
                loss1.backward()
                loss2.backward()
                optimizer1.step()
                optimizer2.step()
                avg_loss1 += loss1.item() / len(train_loader)
                avg_loss2 += loss2.item() / len(train_loader)
            valid_preds1, avg_val_loss1, valid_preds2, avg_val_loss2 = self._val(valid_loader, model1, model2)
            scheduler1.step()
            scheduler2.step()

            self.logger.info("=========================================")
            self.logger.info(f"Epoch {epoch + 1} / {n_epochs}")
            self.logger.info("=========================================")
            self.logger.info(f"avg_loss: {avg_loss1:.8f}")
            self.logger.info(f"avg_val_loss: {avg_val_loss1:.8f}")
            self.logger.info(f"avg_loss: {avg_loss2:.8f}")
            self.logger.info(f"avg_val_loss: {avg_val_loss2:.8f}")
            
            if best_score1 > avg_val_loss1:
                torch.save(model1.state_dict(),
                           self.path1 / f"1best{self.fold_num}.pth")
                self.logger.info(f"Save model at Epoch {epoch + 1}")
                best_score1 = avg_val_loss1
                
            if best_score2 > avg_val_loss2:
                torch.save(model2.state_dict(),
                           self.path2 / f"2best{self.fold_num}.pth")
                self.logger.info(f"Save model at Epoch {epoch + 1}")
                best_score2 = avg_val_loss2
                
        model1.load_state_dict(torch.load(self.path1 / f"1best{self.fold_num}.pth"))
        model2.load_state_dict(torch.load(self.path2 / f"2best{self.fold_num}.pth"))
        
        valid_preds1, avg_val_loss1, valid_preds2, avg_val_loss2 = self._val(valid_loader, model1, model2)
        #print('vpp'+str(valid_preds1.shape[1]))
        #self.logger.info(f"Best Validation Loss: {avg_val_loss:.8f}")
        return valid_preds1, valid_preds2
    
    def _val(self, loader, model1, model2):
        model1.eval()
        model2.eval()
        valid_preds1 = np.zeros((len(loader.dataset), num_classes_c))
        valid_preds2 = np.zeros((len(loader.dataset), num_classes_t))
        avg_val_loss1 = 0.0
        avg_val_loss2 = 0.0
        for i, (i_batch, y_batch) in enumerate(loader):
            with torch.no_grad():
                y_pred1 = model1(i_batch).detach()
                avg_val_loss1 += self.loss_fn(y_pred1, y_batch[0]).item() / len(loader)
                valid_preds1[i * self.valid_batch:(i + 1) * self.valid_batch] = \
                    y_pred1.cpu().numpy()
                y_pred2 = model2(i_batch).detach()
                avg_val_loss2 += self.loss_fn(y_pred2, y_batch[1]).item() / len(loader)
                valid_preds2[i * self.valid_batch:(i + 1) * self.valid_batch] = \
                    y_pred2.cpu().numpy()
        #print('vp1'+str(valid_preds1.shape[1]))
        return valid_preds1, avg_val_loss1, valid_preds2, avg_val_loss2
    
    def predict(self, X):
        #print('pred')
        dataset = IMetDataset(X, labels=None)
        loader = data.DataLoader(dataset, 
                                 batch_size=self.valid_batch, 
                                 shuffle=False)
        model1 = self.model1(**self.kwargs)
        model2 = self.model2(**self.kwargs)
        preds1 = np.zeros((X.size(0), num_classes_c))
        #print(list(self.path1.iterdir()))
        for path in self.path1.iterdir():
            with timer(f"Using {str(path)}", self.logger):
                model1.load_state_dict(torch.load(path))
                model1.to(self.device)
                model1.eval()
                temp1 = np.zeros_like(preds1)
                #print('try')
                for i, (i_batch, ) in enumerate(loader):
                    with torch.no_grad():
                        y_pred1 = model1(i_batch).detach()
                        #print(y_pred1[y_pred1 != 0])
                        temp1[i * self.valid_batch:(i + 1) * self.valid_batch] = \
                            y_pred1.cpu().numpy()
                preds1 += temp1 / self.n_splits
        preds2 = np.zeros((X.size(0), num_classes_t))
        for path in self.path2.iterdir():
            with timer(f"Using {str(path)}", self.logger):
                model2.load_state_dict(torch.load(path))
                model2.to(self.device)
                model2.eval()
                temp2 = np.zeros_like(preds2)
                for i, (i_batch, ) in enumerate(loader):
                    with torch.no_grad():
                        y_pred2 = model2(i_batch).detach()
                        temp2[i * self.valid_batch:(i + 1) * self.valid_batch] = \
                            y_pred2.cpu().numpy()
                preds2 += temp2 / self.n_splits
        return preds1, preds2
In [15]:
trainer = Trainer(MultiLayerPerceptron1, MultiLayerPerceptron2, logger, train_batch=64, kwargs={})
In [16]:
from sklearn.model_selection import train_test_split
y = train.attribute_ids.map(lambda x: x.split()).values
valid_preds1, valid_preds2 = trainer.fit(train_tensor, y, n_epochs=40)
2019-05-19 17:15:34,956 Main INFO Fold 1
85.00% [34/40 17:05<03:01]
71.60% [978/1366 00:18<00:07]
2019-05-19 17:16:04,875 Main INFO =========================================
2019-05-19 17:16:04,876 Main INFO Epoch 1 / 40
2019-05-19 17:16:04,877 Main INFO =========================================
2019-05-19 17:16:04,878 Main INFO avg_loss: 0.02136940
2019-05-19 17:16:04,879 Main INFO avg_val_loss: 0.01234018
2019-05-19 17:16:04,880 Main INFO avg_loss: 0.02631314
2019-05-19 17:16:04,881 Main INFO avg_val_loss: 0.01662904
2019-05-19 17:16:04,895 Main INFO Save model at Epoch 1
2019-05-19 17:16:04,910 Main INFO Save model at Epoch 1
2019-05-19 17:16:34,852 Main INFO =========================================
2019-05-19 17:16:34,853 Main INFO Epoch 2 / 40
2019-05-19 17:16:34,854 Main INFO =========================================
2019-05-19 17:16:34,855 Main INFO avg_loss: 0.01266033
2019-05-19 17:16:34,856 Main INFO avg_val_loss: 0.01234314
2019-05-19 17:16:34,857 Main INFO avg_loss: 0.01714156
2019-05-19 17:16:34,857 Main INFO avg_val_loss: 0.01657524
2019-05-19 17:16:34,874 Main INFO Save model at Epoch 2
2019-05-19 17:17:04,826 Main INFO =========================================
2019-05-19 17:17:04,828 Main INFO Epoch 3 / 40
2019-05-19 17:17:04,828 Main INFO =========================================
2019-05-19 17:17:04,829 Main INFO avg_loss: 0.01250700
2019-05-19 17:17:04,830 Main INFO avg_val_loss: 0.01230654
2019-05-19 17:17:04,832 Main INFO avg_loss: 0.01696453
2019-05-19 17:17:04,833 Main INFO avg_val_loss: 0.01659941
2019-05-19 17:17:04,849 Main INFO Save model at Epoch 3
2019-05-19 17:17:35,347 Main INFO =========================================
2019-05-19 17:17:35,348 Main INFO Epoch 4 / 40
2019-05-19 17:17:35,349 Main INFO =========================================
2019-05-19 17:17:35,350 Main INFO avg_loss: 0.01242394
2019-05-19 17:17:35,351 Main INFO avg_val_loss: 0.01230904
2019-05-19 17:17:35,352 Main INFO avg_loss: 0.01687045
2019-05-19 17:17:35,353 Main INFO avg_val_loss: 0.01655923
2019-05-19 17:17:35,372 Main INFO Save model at Epoch 4
2019-05-19 17:18:05,933 Main INFO =========================================
2019-05-19 17:18:05,934 Main INFO Epoch 5 / 40
2019-05-19 17:18:05,935 Main INFO =========================================
2019-05-19 17:18:05,936 Main INFO avg_loss: 0.01238157
2019-05-19 17:18:05,937 Main INFO avg_val_loss: 0.01230548
2019-05-19 17:18:05,938 Main INFO avg_loss: 0.01679824
2019-05-19 17:18:05,939 Main INFO avg_val_loss: 0.01659835
2019-05-19 17:18:05,956 Main INFO Save model at Epoch 5
2019-05-19 17:18:36,174 Main INFO =========================================
2019-05-19 17:18:36,175 Main INFO Epoch 6 / 40
2019-05-19 17:18:36,176 Main INFO =========================================
2019-05-19 17:18:36,178 Main INFO avg_loss: 0.01232990
2019-05-19 17:18:36,178 Main INFO avg_val_loss: 0.01232468
2019-05-19 17:18:36,179 Main INFO avg_loss: 0.01674187
2019-05-19 17:18:36,180 Main INFO avg_val_loss: 0.01658774
2019-05-19 17:19:06,241 Main INFO =========================================
2019-05-19 17:19:06,242 Main INFO Epoch 7 / 40
2019-05-19 17:19:06,243 Main INFO =========================================
2019-05-19 17:19:06,244 Main INFO avg_loss: 0.01227804
2019-05-19 17:19:06,245 Main INFO avg_val_loss: 0.01231818
2019-05-19 17:19:06,246 Main INFO avg_loss: 0.01668410
2019-05-19 17:19:06,247 Main INFO avg_val_loss: 0.01659353
2019-05-19 17:19:36,351 Main INFO =========================================
2019-05-19 17:19:36,352 Main INFO Epoch 8 / 40
2019-05-19 17:19:36,353 Main INFO =========================================
2019-05-19 17:19:36,354 Main INFO avg_loss: 0.01222262
2019-05-19 17:19:36,354 Main INFO avg_val_loss: 0.01230193
2019-05-19 17:19:36,355 Main INFO avg_loss: 0.01661796
2019-05-19 17:19:36,356 Main INFO avg_val_loss: 0.01660076
2019-05-19 17:19:36,374 Main INFO Save model at Epoch 8
2019-05-19 17:20:06,433 Main INFO =========================================
2019-05-19 17:20:06,434 Main INFO Epoch 9 / 40
2019-05-19 17:20:06,435 Main INFO =========================================
2019-05-19 17:20:06,436 Main INFO avg_loss: 0.01215492
2019-05-19 17:20:06,437 Main INFO avg_val_loss: 0.01233548
2019-05-19 17:20:06,438 Main INFO avg_loss: 0.01654664
2019-05-19 17:20:06,439 Main INFO avg_val_loss: 0.01658827
2019-05-19 17:20:36,652 Main INFO =========================================
2019-05-19 17:20:36,653 Main INFO Epoch 10 / 40
2019-05-19 17:20:36,654 Main INFO =========================================
2019-05-19 17:20:36,655 Main INFO avg_loss: 0.01208397
2019-05-19 17:20:36,656 Main INFO avg_val_loss: 0.01241521
2019-05-19 17:20:36,657 Main INFO avg_loss: 0.01647421
2019-05-19 17:20:36,658 Main INFO avg_val_loss: 0.01659795
2019-05-19 17:21:06,869 Main INFO =========================================
2019-05-19 17:21:06,870 Main INFO Epoch 11 / 40
2019-05-19 17:21:06,871 Main INFO =========================================
2019-05-19 17:21:06,872 Main INFO avg_loss: 0.01199836
2019-05-19 17:21:06,873 Main INFO avg_val_loss: 0.01241741
2019-05-19 17:21:06,874 Main INFO avg_loss: 0.01640495
2019-05-19 17:21:06,875 Main INFO avg_val_loss: 0.01659402
2019-05-19 17:21:37,019 Main INFO =========================================
2019-05-19 17:21:37,020 Main INFO Epoch 12 / 40
2019-05-19 17:21:37,020 Main INFO =========================================
2019-05-19 17:21:37,021 Main INFO avg_loss: 0.01189862
2019-05-19 17:21:37,022 Main INFO avg_val_loss: 0.01241629
2019-05-19 17:21:37,023 Main INFO avg_loss: 0.01632676
2019-05-19 17:21:37,024 Main INFO avg_val_loss: 0.01667211
2019-05-19 17:22:07,038 Main INFO =========================================
2019-05-19 17:22:07,040 Main INFO Epoch 13 / 40
2019-05-19 17:22:07,041 Main INFO =========================================
2019-05-19 17:22:07,042 Main INFO avg_loss: 0.01177809
2019-05-19 17:22:07,043 Main INFO avg_val_loss: 0.01247175
2019-05-19 17:22:07,044 Main INFO avg_loss: 0.01624095
2019-05-19 17:22:07,045 Main INFO avg_val_loss: 0.01664508
2019-05-19 17:22:37,335 Main INFO =========================================
2019-05-19 17:22:37,336 Main INFO Epoch 14 / 40
2019-05-19 17:22:37,337 Main INFO =========================================
2019-05-19 17:22:37,338 Main INFO avg_loss: 0.01165795
2019-05-19 17:22:37,339 Main INFO avg_val_loss: 0.01251516
2019-05-19 17:22:37,340 Main INFO avg_loss: 0.01615101
2019-05-19 17:22:37,342 Main INFO avg_val_loss: 0.01670334
2019-05-19 17:23:08,032 Main INFO =========================================
2019-05-19 17:23:08,033 Main INFO Epoch 15 / 40
2019-05-19 17:23:08,034 Main INFO =========================================
2019-05-19 17:23:08,035 Main INFO avg_loss: 0.01150474
2019-05-19 17:23:08,036 Main INFO avg_val_loss: 0.01261902
2019-05-19 17:23:08,037 Main INFO avg_loss: 0.01605874
2019-05-19 17:23:08,038 Main INFO avg_val_loss: 0.01672570
2019-05-19 17:23:38,226 Main INFO =========================================
2019-05-19 17:23:38,227 Main INFO Epoch 16 / 40
2019-05-19 17:23:38,228 Main INFO =========================================
2019-05-19 17:23:38,229 Main INFO avg_loss: 0.01134252
2019-05-19 17:23:38,229 Main INFO avg_val_loss: 0.01272238
2019-05-19 17:23:38,230 Main INFO avg_loss: 0.01596873
2019-05-19 17:23:38,231 Main INFO avg_val_loss: 0.01675963
2019-05-19 17:24:08,295 Main INFO =========================================
2019-05-19 17:24:08,296 Main INFO Epoch 17 / 40
2019-05-19 17:24:08,297 Main INFO =========================================
2019-05-19 17:24:08,297 Main INFO avg_loss: 0.01115958
2019-05-19 17:24:08,298 Main INFO avg_val_loss: 0.01271302
2019-05-19 17:24:08,299 Main INFO avg_loss: 0.01586277
2019-05-19 17:24:08,300 Main INFO avg_val_loss: 0.01683678
2019-05-19 17:24:38,480 Main INFO =========================================
2019-05-19 17:24:38,481 Main INFO Epoch 18 / 40
2019-05-19 17:24:38,482 Main INFO =========================================
2019-05-19 17:24:38,483 Main INFO avg_loss: 0.01097008
2019-05-19 17:24:38,484 Main INFO avg_val_loss: 0.01294305
2019-05-19 17:24:38,485 Main INFO avg_loss: 0.01576247
2019-05-19 17:24:38,486 Main INFO avg_val_loss: 0.01682468
2019-05-19 17:25:08,452 Main INFO =========================================
2019-05-19 17:25:08,453 Main INFO Epoch 19 / 40
2019-05-19 17:25:08,454 Main INFO =========================================
2019-05-19 17:25:08,454 Main INFO avg_loss: 0.01077955
2019-05-19 17:25:08,455 Main INFO avg_val_loss: 0.01298732
2019-05-19 17:25:08,456 Main INFO avg_loss: 0.01565235
2019-05-19 17:25:08,457 Main INFO avg_val_loss: 0.01693498
2019-05-19 17:25:38,525 Main INFO =========================================
2019-05-19 17:25:38,526 Main INFO Epoch 20 / 40
2019-05-19 17:25:38,527 Main INFO =========================================
2019-05-19 17:25:38,528 Main INFO avg_loss: 0.01056819
2019-05-19 17:25:38,529 Main INFO avg_val_loss: 0.01307504
2019-05-19 17:25:38,530 Main INFO avg_loss: 0.01554898
2019-05-19 17:25:38,531 Main INFO avg_val_loss: 0.01688967
2019-05-19 17:26:08,620 Main INFO =========================================
2019-05-19 17:26:08,621 Main INFO Epoch 21 / 40
2019-05-19 17:26:08,622 Main INFO =========================================
2019-05-19 17:26:08,623 Main INFO avg_loss: 0.01036019
2019-05-19 17:26:08,624 Main INFO avg_val_loss: 0.01323750
2019-05-19 17:26:08,625 Main INFO avg_loss: 0.01543477
2019-05-19 17:26:08,626 Main INFO avg_val_loss: 0.01699137
2019-05-19 17:26:38,870 Main INFO =========================================
2019-05-19 17:26:38,871 Main INFO Epoch 22 / 40
2019-05-19 17:26:38,872 Main INFO =========================================
2019-05-19 17:26:38,873 Main INFO avg_loss: 0.01016902
2019-05-19 17:26:38,873 Main INFO avg_val_loss: 0.01336211
2019-05-19 17:26:38,874 Main INFO avg_loss: 0.01532836
2019-05-19 17:26:38,875 Main INFO avg_val_loss: 0.01702198
2019-05-19 17:27:08,915 Main INFO =========================================
2019-05-19 17:27:08,916 Main INFO Epoch 23 / 40
2019-05-19 17:27:08,917 Main INFO =========================================
2019-05-19 17:27:08,918 Main INFO avg_loss: 0.00997221
2019-05-19 17:27:08,919 Main INFO avg_val_loss: 0.01343334
2019-05-19 17:27:08,919 Main INFO avg_loss: 0.01521381
2019-05-19 17:27:08,920 Main INFO avg_val_loss: 0.01702891
2019-05-19 17:27:39,050 Main INFO =========================================
2019-05-19 17:27:39,051 Main INFO Epoch 24 / 40
2019-05-19 17:27:39,052 Main INFO =========================================
2019-05-19 17:27:39,053 Main INFO avg_loss: 0.00977662
2019-05-19 17:27:39,054 Main INFO avg_val_loss: 0.01354172
2019-05-19 17:27:39,055 Main INFO avg_loss: 0.01511677
2019-05-19 17:27:39,056 Main INFO avg_val_loss: 0.01708103
2019-05-19 17:28:09,930 Main INFO =========================================
2019-05-19 17:28:09,932 Main INFO Epoch 25 / 40
2019-05-19 17:28:09,932 Main INFO =========================================
2019-05-19 17:28:09,933 Main INFO avg_loss: 0.00960736
2019-05-19 17:28:09,934 Main INFO avg_val_loss: 0.01376129
2019-05-19 17:28:09,935 Main INFO avg_loss: 0.01500759
2019-05-19 17:28:09,936 Main INFO avg_val_loss: 0.01708940
2019-05-19 17:28:40,046 Main INFO =========================================
2019-05-19 17:28:40,047 Main INFO Epoch 26 / 40
2019-05-19 17:28:40,048 Main INFO =========================================
2019-05-19 17:28:40,049 Main INFO avg_loss: 0.00943622
2019-05-19 17:28:40,049 Main INFO avg_val_loss: 0.01381716
2019-05-19 17:28:40,050 Main INFO avg_loss: 0.01490993
2019-05-19 17:28:40,051 Main INFO avg_val_loss: 0.01718958
2019-05-19 17:29:10,124 Main INFO =========================================
2019-05-19 17:29:10,126 Main INFO Epoch 27 / 40
2019-05-19 17:29:10,127 Main INFO =========================================
2019-05-19 17:29:10,128 Main INFO avg_loss: 0.00928547
2019-05-19 17:29:10,129 Main INFO avg_val_loss: 0.01398815
2019-05-19 17:29:10,131 Main INFO avg_loss: 0.01482284
2019-05-19 17:29:10,132 Main INFO avg_val_loss: 0.01711607
2019-05-19 17:29:40,185 Main INFO =========================================
2019-05-19 17:29:40,186 Main INFO Epoch 28 / 40
2019-05-19 17:29:40,187 Main INFO =========================================
2019-05-19 17:29:40,188 Main INFO avg_loss: 0.00913265
2019-05-19 17:29:40,189 Main INFO avg_val_loss: 0.01406358
2019-05-19 17:29:40,190 Main INFO avg_loss: 0.01474048
2019-05-19 17:29:40,191 Main INFO avg_val_loss: 0.01731832
2019-05-19 17:30:10,195 Main INFO =========================================
2019-05-19 17:30:10,196 Main INFO Epoch 29 / 40
2019-05-19 17:30:10,197 Main INFO =========================================
2019-05-19 17:30:10,198 Main INFO avg_loss: 0.00899717
2019-05-19 17:30:10,199 Main INFO avg_val_loss: 0.01416660
2019-05-19 17:30:10,200 Main INFO avg_loss: 0.01466195
2019-05-19 17:30:10,201 Main INFO avg_val_loss: 0.01730115
2019-05-19 17:30:40,313 Main INFO =========================================
2019-05-19 17:30:40,314 Main INFO Epoch 30 / 40
2019-05-19 17:30:40,315 Main INFO =========================================
2019-05-19 17:30:40,316 Main INFO avg_loss: 0.00887910
2019-05-19 17:30:40,317 Main INFO avg_val_loss: 0.01422593
2019-05-19 17:30:40,317 Main INFO avg_loss: 0.01458952
2019-05-19 17:30:40,318 Main INFO avg_val_loss: 0.01734348
2019-05-19 17:31:10,518 Main INFO =========================================
2019-05-19 17:31:10,520 Main INFO Epoch 31 / 40
2019-05-19 17:31:10,520 Main INFO =========================================
2019-05-19 17:31:10,521 Main INFO avg_loss: 0.00878750
2019-05-19 17:31:10,522 Main INFO avg_val_loss: 0.01431212
2019-05-19 17:31:10,523 Main INFO avg_loss: 0.01452598
2019-05-19 17:31:10,524 Main INFO avg_val_loss: 0.01737726
2019-05-19 17:31:40,663 Main INFO =========================================
2019-05-19 17:31:40,664 Main INFO Epoch 32 / 40
2019-05-19 17:31:40,665 Main INFO =========================================
2019-05-19 17:31:40,666 Main INFO avg_loss: 0.00869077
2019-05-19 17:31:40,667 Main INFO avg_val_loss: 0.01436828
2019-05-19 17:31:40,668 Main INFO avg_loss: 0.01446899
2019-05-19 17:31:40,669 Main INFO avg_val_loss: 0.01738612
2019-05-19 17:32:10,617 Main INFO =========================================
2019-05-19 17:32:10,619 Main INFO Epoch 33 / 40
2019-05-19 17:32:10,619 Main INFO =========================================
2019-05-19 17:32:10,620 Main INFO avg_loss: 0.00861716
2019-05-19 17:32:10,621 Main INFO avg_val_loss: 0.01443698
2019-05-19 17:32:10,622 Main INFO avg_loss: 0.01442813
2019-05-19 17:32:10,623 Main INFO avg_val_loss: 0.01746377
2019-05-19 17:32:40,751 Main INFO =========================================
2019-05-19 17:32:40,752 Main INFO Epoch 34 / 40
2019-05-19 17:32:40,753 Main INFO =========================================
2019-05-19 17:32:40,754 Main INFO avg_loss: 0.00854192
2019-05-19 17:32:40,755 Main INFO avg_val_loss: 0.01453591
2019-05-19 17:32:40,756 Main INFO avg_loss: 0.01438324
2019-05-19 17:32:40,757 Main INFO avg_val_loss: 0.01743537
In [17]:
def threshold_search(y_pred, y_true):
    score = []
    candidates = np.arange(0, 1.0, 0.01)
    for th in progress_bar(candidates):
        yp = (y_pred > th).astype(int)
        score.append(fbeta_score(y_pred=yp, y_true=y_true, beta=2, average="samples"))
    score = np.array(score)
    pm = score.argmax()
    best_th, best_score = candidates[pm], score[pm]
    return best_th, best_score
In [18]:
y_true = np.zeros((train.shape[0], 1103)).astype(int)
for i, row in enumerate(y):
    for idx in row:
        y_true[i, int(idx)] = 1
In [19]:
best_threshold1, best_score1 = threshold_search(valid_preds1, targets_c)
best_score1
best_threshold2, best_score2 = threshold_search(valid_preds2, targets_t)
best_score2
100.00% [100/100 05:29<00:00]
/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.
  'precision', 'predicted', average, warn_for)
100.00% [100/100 10:06<00:00]
Out[19]:
0.21991223887764777
In [20]:
test_preds1, test_preds2  = trainer.predict(test_tensor)
2019-05-19 19:11:49,688 Main INFO [Using bin1/2019-05-19-17-15-34/1best4.pth] done in 0.11359095573425293 s
2019-05-19 19:11:49,786 Main INFO [Using bin1/2019-05-19-17-15-34/1best0.pth] done in 0.09763526916503906 s
2019-05-19 19:11:49,890 Main INFO [Using bin1/2019-05-19-17-15-34/1best3.pth] done in 0.10192990303039551 s
2019-05-19 19:11:49,986 Main INFO [Using bin1/2019-05-19-17-15-34/1best2.pth] done in 0.09511685371398926 s
2019-05-19 19:11:50,085 Main INFO [Using bin1/2019-05-19-17-15-34/1best1.pth] done in 0.09669661521911621 s
2019-05-19 19:11:50,264 Main INFO [Using bin2/2019-05-19-17-15-34/2best0.pth] done in 0.17796993255615234 s
2019-05-19 19:11:50,408 Main INFO [Using bin2/2019-05-19-17-15-34/2best4.pth] done in 0.14308691024780273 s
2019-05-19 19:11:50,556 Main INFO [Using bin2/2019-05-19-17-15-34/2best3.pth] done in 0.14675283432006836 s
2019-05-19 19:11:50,702 Main INFO [Using bin2/2019-05-19-17-15-34/2best2.pth] done in 0.14363956451416016 s
2019-05-19 19:11:50,852 Main INFO [Using bin2/2019-05-19-17-15-34/2best1.pth] done in 0.14873361587524414 s
In [21]:
preds1 = (test_preds1 > best_threshold1).astype(int)
preds2 = (test_preds2 > best_threshold2).astype(int)
In [22]:
prediction = []
for i in range(preds1.shape[0]):
    pred1 = [i for i in np.argwhere(preds1[i] == 1.0).reshape(-1).tolist() if i != (num_classes_c - 1)]
    pred2 = [(i + num_classes_c - 1) for i in np.argwhere(preds2[i] == 1.0).reshape(-1).tolist() if i != (num_classes_c + num_classes_t - 2)]
    pred_str = " ".join(list(map(str, pred1 + pred2)))
    prediction.append(pred_str)
#print(test_preds1[test_preds1 != 0])
sample.attribute_ids = prediction
sample.to_csv("submission.csv", index=False)
sample.head()
Out[22]:
id attribute_ids
0 10023b2cc4ed5f68 13 121 147 189 369 813 1039 1059 1092
1 100fbe75ed8fd887 13 121 147 369 671 813 1039 1059 1092
2 101b627524a04f19 121 147 189 813 1059 1092
3 10234480c41284c6 13 51 147 189 480 483 501 671 737 738 776 813 ...
4 1023b0e2636dcea8 51 79 147 189 671 780 813 1059 1092