imet_atk_task2_1 densenet121

From: https://www.kaggle.com/icekstry/imet-atk-task2-1-densenet121

Author: xutao

In [1]:
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory
import os
print(os.listdir("../input"))
# Any results you write to the current directory are saved as output.

# 以下内容参考于:https://www.kaggle.com/hidehisaarai1213/imet-pytorch-starter
# imet,densenet121

import gc
import sys
import time 
import random
import logging
import datetime as dt

import torch
import torch.nn as nn
import torch.optim as optim
import torch.utils.data as data
from torch.utils.data import TensorDataset,DataLoader,Dataset
import torch.nn.functional as F
import torchvision as vision

from torch.optim.lr_scheduler import CosineAnnealingLR
from pathlib import Path
from PIL import Image
from contextlib import contextmanager

from joblib import Parallel,delayed
from tqdm import tqdm
from fastprogress import master_bar, progress_bar

from sklearn.model_selection import KFold
from sklearn.metrics import fbeta_score

torch.multiprocessing.set_start_method("spawn")
['imet-2019-fgvc6', 'imet-pytorch-starter']
In [2]:
@contextmanager
def timer(name="Main", logger=None):
    t0 = time.time()
    yield
    msg = f"[{name}] done in {time.time() - t0} s"
    if logger is not None:
        logger.info(msg)
    else:
        print(msg)

def get_logger(name="Main", tag="exp", log_dir="log/"):
    log_path = Path(log_dir)
    path = log_path / tag
    path.mkdir(exist_ok=True, parents=True)
    
    logger = logging.getLogger(name)
    logger.setLevel(logging.INFO)
    
    fh = logging.FileHandler( path / (dt.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + ".log"))
    sh = logging.StreamHandler(sys.stdout)
    formatter = logging.Formatter("%(asctime)s %(name)s %(levelname)s %(message)s")
    
    fh.setFormatter(formatter)
    sh.setFormatter(formatter)
    logger.addHandler(fh)
    logger.addHandler(sh)
    return logger

# fixed seed
def seed_torch(seed=1029):
    random.seed(seed)
    os.environ["PYTHONHASHSEED"] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    torch.backends.cudnn.deterministic = True
In [3]:
logger = get_logger(name="Main", tag="Pytorch-VGG16")
# import the pretrained net
!ls ../input/imet-pytorch-starter/

# input data
labels = pd.read_csv("../input/imet-2019-fgvc6/labels.csv")
train = pd.read_csv("../input/imet-2019-fgvc6/train.csv")
sample = pd.read_csv("../input/imet-2019-fgvc6/sample_submission.csv")
train.head()
__notebook__.ipynb  bin		     densenet201.pth  resnet50.pth
__output__.json     custom.css	     log	      submission.csv
__results__.html    densenet121.pth  resnet34.pth
Out[3]:
id attribute_ids
0 1000483014d91860 147 616 813
1 1000fe2e667721fe 51 616 734 813
2 1001614cb89646ee 776
3 10041eb49b297c08 51 671 698 813 1092
4 100501c227f8beea 13 404 492 903 1093
In [4]:
# load data
class ImageDataLoader(data.DataLoader):
    
    def __init__(self, root_dir:Path, df:pd.DataFrame, mode="train", transforms=None):
        self._root = root_dir
        self.transform = transforms[mode]
        self._img_id = (df["id"] + ".png").values  #image = image_id + .png
    
    def __len__(self):
        return len(self._img_id)
    
    def __getitem__(self, idx):
        img_id = self._img_id[idx]
        file_name = self._root / img_id
        img = Image.open(file_name)
        if self.transform:
            img = self.transform(img)        
        return [img]

# define data_transforms
data_transforms = {
    "train": vision.transforms.Compose([
        vision.transforms.RandomResizedCrop(224),
        vision.transforms.RandomHorizontalFlip(),
        vision.transforms.ToTensor(),
        vision.transforms.Normalize(
            [0.485, 0.455, 0.406],
            [0.229, 0.224, 0.225])
    ]),

    "val":vision.transforms.Compose([
        vision.transforms.Resize(256),
        vision.transforms.CenterCrop(224),
        vision.transforms.ToTensor(),
        vision.transforms.Normalize(
            [0.485, 0.455, 0.406],
            [0.229, 0.224, 0.225])
    ])
}

# what the meaning of below line code:
data_transforms["test"] = data_transforms["val"]
In [5]:
# This loader is to be used for serving image tensors for the MLP
class IMetDataset(Dataset):
    def __init__(self, tensor, device="cuda:0", labels=None):
        self.tensor = tensor
        self.labels = labels
        self.device = device
    
    def __len__(self):
        return self.tensor.size(0)
        
    def __getitem__(self, idx):
        tensor = self.tensor[idx, :]
        if self.labels is not None:
            label = self.labels[idx]
            label_tensor = torch.zeros(1, 1103)  # numbers of label is 1103
            for i in label:
                label_tensor[0, int(i)] = 1
            label_tensor = label_tensor.to(self.device)
            return [tensor, label_tensor]
        else:
            return [tensor]
In [6]:
print(IMetDataset)
<class '__main__.IMetDataset'>
In [7]:
# Model
class Classifier(nn.Module):
    def __init__(self):
        super(Classifier, self).__init__()
    
    def forward(self, x):
        return x

class Densenet121(nn.Module):
    def __init__(self, pretrained: Path):
        super(Densenet121, self).__init__()
        self.densenet121 = vision.models.densenet121()
        self.densenet121.load_state_dict(torch.load(pretrained))
        self.densenet121.classifier = Classifier()
        dense = nn.Sequential(*list(self.densenet121.children())[:-1])
        for param in dense.parameters():
            param.requires_gard = False
    
    def forward(self, x):
        return self.densenet121(x)

class MultiLayerPerceptron(nn.Module):
    def __init__(self):
        super(MultiLayerPerceptron, self).__init__()
        self.linear1 = nn.Linear(1024, 1024)
        self.relu = nn.ReLU()
        self.linear2 = nn.Linear(1024, 1103)
        self.dropout = nn.Dropout(0.5)
        self.sigmoid = nn.Sigmoid()
        
    def forward(self, x):
        x = self.relu(self.linear1(x))
        x = self.dropout(x)
        return self.sigmoid(self.linear2(x))
In [8]:
# Data Preprocessing with Densenet121
train_dataset = ImageDataLoader(
    root_dir=Path("../input/imet-2019-fgvc6/train/"),
    df=train,
    mode="train",
    transforms=data_transforms )
    
train_loader = data.DataLoader(dataset=train_dataset,shuffle=False,batch_size=32)

test_dataset = ImageDataLoader(root_dir=Path("../input/imet-2019-fgvc6/test/"),
    df=sample,
    mode="test",
    transforms=data_transforms)

test_loader = data.DataLoader(dataset=test_dataset,shuffle=False,batch_size=32)
In [9]:
def get_feature_vector(df, loader, device):
    matrix = torch.zeros((df.shape[0], 1024)).to(device)
    model = Densenet121("../input/imet-pytorch-starter/densenet121.pth")
    model.to(device)
    batch = loader.batch_size
    for i, (i_batch,) in tqdm(enumerate(loader)):
        i_batch = i_batch.to(device)
        pred = model(i_batch).detach()
        matrix[i * batch:(i+1) * batch] = pred
    return matrix
    
train_tensor = get_feature_vector(train, train_loader,"cuda:0")
test_tensor = get_feature_vector(sample, test_loader, "cuda:0")

del train_dataset, train_loader
del test_dataset, test_loader
gc.collect()
3414it [18:31,  3.15it/s]
233it [01:20,  3.23it/s]
Out[9]:
371
In [10]:
# Train Utilities
class Trainer:
    def __init__(self, model, logger, n_splits=5, seed=42, device="cuda:0",
        train_batch=32, valid_batch=64, kwargs={}):
        self.model = model
        self.logger = logger
        self.device = device
        self.n_splits = n_splits
        self.seed = seed
        self.train_batch = train_batch
        self.valid_batch = valid_batch
        self.kwargs = kwargs
        
        self.best_score = None
        self.tag = dt.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
        self.loss_fn = nn.BCELoss(reduction="mean").to(self.device) # binal cross entropy
        path = Path(f"bin/{self.tag}")
        path.mkdir(exist_ok=True, parents=True)
        self.path = path
    
    def fit(self, X, y, n_epochs=10):
        train_preds = np.zeros((len(X), 1103))
        fold = KFold(n_splits=self.n_splits, random_state=self.seed)
        for i, (trn_idx, val_idx) in enumerate(fold.split(X)):
            self.fold_num=i
            self.logger.info(f"Fold {i + 1}")
            
            X_train, X_val = X[trn_idx, :], X[val_idx,:]
            y_train, y_val = y[trn_idx], y[val_idx]
            valid_preds = self._fit(X_train, y_train, X_val, y_val, n_epochs)
            train_preds[val_idx] = valid_preds
        return train_preds
    
    def _fit(self, X_train, y_train, X_val, y_val, n_epochs):
        seed_torch(self.seed)
        train_dataset = IMetDataset(X_train, labels=y_train, device=self.device)
        train_loader = data.DataLoader(train_dataset,batch_size=self.train_batch,shuffle=True)
        valid_dataset = IMetDataset(X_val, labels=y_val,device=self.device)
        valid_loader = data.DataLoader(valid_dataset,batch_size=self.valid_batch,shuffle=False)
        model = self.model(**self.kwargs)
        model.to(self.device)
        optimizer = optim.Adam(params=model.parameters(), lr=0.0001)
        scheduler = CosineAnnealingLR(optimizer, T_max=n_epochs)
        best_score = np.inf
        mb = master_bar(range(n_epochs))
        for epoch in mb:
            model.train()
            avg_loss = 0.0
            for i_batch, y_batch in progress_bar(train_loader, parent=mb):
                y_pred = model(i_batch)
                loss = self.loss_fn(y_pred, y_batch)
                optimizer.zero_grad()
                loss.backward()
                optimizer.step()
                avg_loss += loss.item() / len(train_loader)
                valid_preds, avg_val_loss = self._val(valid_loader, model)
                
                scheduler.step()
                self.logger.info("========================================")
                self.logger.info(f"Epoch {epoch + 1} / {n_epochs}")
                self.logger.info("========================================")
                self.logger.info(f"avg_loss: {avg_loss:.8f}")
                self.logger.info(f"avg_val_loss: {avg_val_loss:.8f}")
                
                if best_score > avg_val_loss:
                    torch.save(model.state_dict(),self.path / f"best{self.fold_num}.pth")
                    self.logger.info(f"Save model at Epoch {epoch +1}")
                    best_socre = avg_val_loss
                    model.load_state_dict(torch.load(self.path/f"best{self.fold_num}.pth" ))
                    valid_preds, avg_val_loss = self._val(valid_loader, model)
                    self.logger.info(f"Best Validation Loss:{avg_val_loss:.8f}")
        return preds      
    
    def _val(self, loader, model):
        model.eval()
        valid_preds = np.zeros((len(loader.dataset), 1103))
        avg_val_loss = 0.0
        for i, (i_batch, y_batch) in enumerate(loader):
            with torch.no_grad():
                y_pred = model(i_batch).detach()
                avg_val_loss += self.loss_fn(y_pred, y_batch).item() / len(loader)
                valid_preds[i * self.valid_batch:(i + 1) * self.valid_batch] = \
                    y_pred.cpu().numpy()
        return valid_preds, avg_val_loss
In [11]:
# Training 
trainer = Trainer(MultiLayerPerceptron, logger, train_batch=32,kwargs={})
y = train.attribute_ids.map(lambda x: x.split()).values
In [12]:
valid_preds = Trainer.fit(train_tensor, y, n_epochs=5)
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-12-89ecc22f3aef> in <module>()
----> 1 valid_preds = Trainer.fit(train_tensor, y, n_epochs=5)

TypeError: fit() missing 1 required positional argument: 'y'
In [13]:
def threshold_search(y_pred, y_true):
    score = []
    candidates = np.arange(0, 1.0, 0.01)
    for th in progress_bar(candidates):
        yp = (y_pred > th).astype(int)
        score.append(fbeta_score(y_pred=yp, y_true=y_true, beta=2, average="samples"))
    score = np.array(score)
    pm = score.argmax()
    best_th, best_score = candidates[pm], score[pm]
    return best_th, best_score

y_true = np.zeros((train.shape[0], 1103)).astype(int)
for i, row in enumerate(y):
    for idx in row:
        y_true[i, int(idx)] = 1
        
best_threshold, best_score = threshold_search(valid_preds, y_true)
best_score
---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-13-3e7496f08b9e> in <module>()
     15         y_true[i, int(idx)] = 1
     16 
---> 17 best_threshold, best_score = threshold_search(valid_preds, y_true)
     18 best_score

NameError: name 'valid_preds' is not defined
In [14]:
# prediction testdataset
test_preds = trainer.predict(test_tensor)
preds = (test_preds > best_threshold).astype(int)
prediction = []
for i in range(preds.shape[0]):
    pred1 = np.argwhere(preds[i] == 1.0).reshape(-1).tolist()
    pred_str = " ".join(list(map(str, pred1)))
    prediction.append(pred_str)
    
sample.attribute_ids = prediction
sample.to_csv("submission.csv", index=False)
sample.head()
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-14-64e0c5440ac7> in <module>()
      1 # prediction testdataset
----> 2 test_preds = trainer.predict(test_tensor)
      3 preds = (test_preds > best_threshold).astype(int)
      4 prediction = []
      5 for i in range(preds.shape[0]):

AttributeError: 'Trainer' object has no attribute 'predict'