Keras Experimentation

From: https://www.kaggle.com/mercurialmonkey/keras-experimentation

Author: MercurialMonkey

Simple example of transfer learning from pretrained model using Keras.

  • Loss: Focal loss
  • Metrics: f2_score
In [1]:
import os
import cv2
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import json
from keras.models import Sequential, Model
from keras.layers import Dense, Flatten, Activation, Dropout, GlobalAveragePooling2D
from keras.preprocessing.image import ImageDataGenerator
from keras import optimizers, applications
from keras.callbacks import ModelCheckpoint, LearningRateScheduler, TensorBoard, EarlyStopping
from keras import backend as K 
Using TensorFlow backend.
In [2]:
train_df = pd.read_csv("../input/imet-2019-fgvc6/train.csv")
train_df["attribute_ids"]=train_df["attribute_ids"].apply(lambda x:x.split(" "))
train_df["id"]=train_df["id"].apply(lambda x:x+".png")
train_df.head()
Out[2]:
id attribute_ids
0 1000483014d91860.png [147, 616, 813]
1 1000fe2e667721fe.png [51, 616, 734, 813]
2 1001614cb89646ee.png [776]
3 10041eb49b297c08.png [51, 671, 698, 813, 1092]
4 100501c227f8beea.png [13, 404, 492, 903, 1093]
In [3]:
label_df = pd.read_csv("../input/imet-2019-fgvc6/labels.csv")
print(label_df.shape)
label_df.head()
(1103, 2)
Out[3]:
attribute_id attribute_name
0 0 culture::abruzzi
1 1 culture::achaemenid
2 2 culture::aegean
3 3 culture::afghan
4 4 culture::after british
In [4]:
# Example of images with tags

i = 1
plt.figure(figsize=[30,30])
for img_name in os.listdir("../input/imet-2019-fgvc6/train/")[5:10]:   
    img = cv2.imread("../input/imet-2019-fgvc6/train/%s" % img_name)[...,[2, 1, 0]]
    plt.subplot(5, 1, i)
    plt.imshow(img)
    ids = train_df[train_df["id"] == img_name]["attribute_ids"]
    title_val = []
    for tag_id in ids.values[0]:
        att_name = label_df[label_df['attribute_id'].astype(str) == tag_id]['attribute_name'].values[0]
        title_val.append(att_name)
    plt.title(title_val)
    i += 1
    
plt.show()
In [5]:
nb_classes = 1103
batch_size = 300
img_size = 75
nb_epochs = 25
In [6]:
lbls = list(map(str, range(nb_classes)))
In [7]:
%%time

train_datagen=ImageDataGenerator(
    rescale=1./255, 
    validation_split=0.25,
    horizontal_flip = True,    
    zoom_range = 0.3,
    width_shift_range = 0.3,
    height_shift_range=0.3
    )

train_generator=train_datagen.flow_from_dataframe(
    dataframe=train_df,
    directory="../input/imet-2019-fgvc6/train",
    x_col="id",
    y_col="attribute_ids",
    batch_size=batch_size,
    shuffle=True,
    class_mode="categorical",
    classes=lbls,
    target_size=(img_size,img_size),
    subset='training')

valid_generator=train_datagen.flow_from_dataframe(
    dataframe=train_df,
    directory="../input/imet-2019-fgvc6/train",
    x_col="id",
    y_col="attribute_ids",
    batch_size=batch_size,
    shuffle=True,
    class_mode="categorical",    
    classes=lbls,
    target_size=(img_size,img_size),
    subset='validation')
Found 81928 images belonging to 1103 classes.
Found 27309 images belonging to 1103 classes.
CPU times: user 6.88 s, sys: 8.15 s, total: 15 s
Wall time: 1min 13s
In [8]:
# Loss

gamma = 2.0
epsilon = K.epsilon()
def focal_loss(y_true, y_pred):
    pt = y_pred * y_true + (1-y_pred) * (1-y_true)
    pt = K.clip(pt, epsilon, 1-epsilon)
    CE = -K.log(pt)
    FL = K.pow(1-pt, gamma) * CE
    loss = K.sum(FL, axis=1)
    return loss
In [9]:
# Metric

def f2_score(y_true, y_pred):
    beta = 2
    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)), axis=1)
    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)), axis=1)
    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)), axis=1)
    
    precision = true_positives / (predicted_positives + K.epsilon())
    recall = true_positives / (possible_positives + K.epsilon())
    
    return K.mean(((1+beta**2)*precision*recall) / ((beta**2)*precision+recall+K.epsilon()))
In [10]:
model = applications.InceptionResNetV2(weights=None, 
                          include_top=False, 
                          input_shape=(img_size, img_size, 3))
model.load_weights('../input/inceptionresnetv2/inception_resnet_v2_weights_tf_dim_ordering_tf_kernels_notop.h5')
WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
In [11]:
model.trainable = False
In [12]:
# Freeze some layers
# for layer in model.layers[:-4]:
#     layer.trainable = False
In [13]:
#Adding custom layers 
x = model.output
x = Flatten()(x)
x = Dense(1024, activation="relu")(x)
x = Dropout(0.5)(x)
predictions = Dense(nb_classes, activation="softmax")(x)
model_final = Model(input = model.input, output = predictions)

model_final.compile(optimizers.rmsprop(lr=0.001, decay=1e-6),loss=focal_loss,metrics=[f2_score])
WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.
Instructions for updating:
Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.
/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:7: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=Tensor("in..., outputs=Tensor("de...)`
  import sys
In [14]:
# model_final.summary()
In [15]:
# Callbacks

checkpoint = ModelCheckpoint("model_1.h5", monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False, mode='auto', period=1)
early = EarlyStopping(monitor='val_loss', min_delta=0, patience=5, verbose=1, mode='auto')
In [16]:
%%time
history = model_final.fit_generator(generator=train_generator,                   
                                    steps_per_epoch=500,
                                    validation_data=valid_generator,                    
                                    validation_steps=200,
                                    epochs=nb_epochs,
                                    callbacks = [checkpoint, early],
                                    max_queue_size=16,
                                    workers=2,
                                    use_multiprocessing=True,
                                    verbose=0)
WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
/opt/conda/lib/python3.6/site-packages/keras/engine/training_generator.py:47: UserWarning: Using a generator with `use_multiprocessing=True` and multiple workers may duplicate your data. Please consider using the`keras.utils.Sequence class.
  UserWarning('Using a generator with `use_multiprocessing=True`'
Epoch 00001: val_loss improved from inf to 21.47183, saving model to model_1.h5

Epoch 00002: val_loss improved from 21.47183 to 13.86855, saving model to model_1.h5

Epoch 00003: val_loss improved from 13.86855 to 13.50887, saving model to model_1.h5

Epoch 00004: val_loss did not improve from 13.50887

Epoch 00005: val_loss did not improve from 13.50887

Epoch 00006: val_loss improved from 13.50887 to 13.04655, saving model to model_1.h5

Epoch 00007: val_loss improved from 13.04655 to 12.99300, saving model to model_1.h5

Epoch 00008: val_loss improved from 12.99300 to 12.58820, saving model to model_1.h5

Epoch 00009: val_loss did not improve from 12.58820

Epoch 00010: val_loss improved from 12.58820 to 12.46322, saving model to model_1.h5

Epoch 00011: val_loss did not improve from 12.46322

Epoch 00012: val_loss improved from 12.46322 to 12.11025, saving model to model_1.h5

Epoch 00013: val_loss did not improve from 12.11025

Epoch 00014: val_loss did not improve from 12.11025

Epoch 00015: val_loss did not improve from 12.11025

Epoch 00016: val_loss did not improve from 12.11025

Epoch 00017: val_loss did not improve from 12.11025
Epoch 00017: early stopping
CPU times: user 1h 5min 3s, sys: 15min 21s, total: 1h 20min 24s
Wall time: 5h 47min 29s
In [17]:
with open('history.json', 'w') as f:
    json.dump(history.history, f)

history_df = pd.DataFrame(history.history)
history_df[['loss', 'val_loss']].plot()
history_df[['f2_score', 'val_f2_score']].plot()
Out[17]:
<matplotlib.axes._subplots.AxesSubplot at 0x7f8a5d61e438>
In [18]:
sam_sub_df = pd.read_csv('../input/imet-2019-fgvc6/sample_submission.csv')
sam_sub_df["id"]=sam_sub_df["id"].apply(lambda x:x+".png")
print(sam_sub_df.shape)
sam_sub_df.head()
(38801, 2)
Out[18]:
id attribute_ids
0 1002e7c8e9172e5c.png 0 1 2
1 10045628394b0c85.png 0 1 2
2 1006c05aacbeaea6.png 0 1 2
3 10073dbee72a9e88.png 0 1 2
4 10077d92747afb1a.png 0 1 2
In [19]:
%%time
test_datagen = ImageDataGenerator(rescale=1./255)
test_generator = test_datagen.flow_from_dataframe(  
        dataframe=sam_sub_df,
        directory = "../input/imet-2019-fgvc6/test",    
        x_col="id",
        target_size = (img_size,img_size),
        batch_size = 1,
        shuffle = False,
        class_mode = None
        )
Found 38801 images.
CPU times: user 932 ms, sys: 2.12 s, total: 3.05 s
Wall time: 14.1 s
In [20]:
%%time
test_generator.reset()
predict = model_final.predict_generator(test_generator, steps = len(test_generator.filenames))
CPU times: user 1h 22min 25s, sys: 47.8 s, total: 1h 23min 12s
Wall time: 57min 11s
In [21]:
len(predict)
Out[21]:
38801
In [22]:
%%time
import operator
predicted_class_indices_3=[]
for i in range(len(predict)):         
    d = {}
    for index, value in enumerate(predict[i]):               
        if value > 0.03:            
            d[index] = value 
    sorted_d = sorted(d.items(), key=operator.itemgetter(1), reverse=True)
    
    # Take only first 10 items
    predicted_class_indices_3.append([i[0] for i in sorted_d[:10]])
CPU times: user 1min 18s, sys: 0 ns, total: 1min 18s
Wall time: 1min 18s
In [23]:
%%time
predictions_3=[]

for i in range(len(predicted_class_indices_3)):
    labels = (train_generator.class_indices)
    labels = dict((v,k) for k,v in labels.items())
    predictions = [labels[k] for k in predicted_class_indices_3[i]]
    predictions_3.append(predictions)
CPU times: user 6.89 s, sys: 0 ns, total: 6.89 s
Wall time: 6.9 s
In [24]:
predict_3 = []
for i in range(len(predictions_3)):
    str3 = " ".join(predictions_3[i])
    predict_3.append(str3)
In [25]:
filenames=test_generator.filenames
results=pd.DataFrame({"id":filenames,
                      "attribute_ids":predict_3})
results['id'] = results['id'].map(lambda x: str(x)[:-4])
results.to_csv("submission.csv",index=False)
In [26]:
results.head()
Out[26]:
id attribute_ids
0 1002e7c8e9172e5c 147 1034 121 1098 950 189
1 10045628394b0c85 813 1092 744 147 51 189 896 903
2 1006c05aacbeaea6 813 1092 147 51 671 744 13 780 194
3 10073dbee72a9e88 634 125 495 754 161 872
4 10077d92747afb1a 156 189 671 147 780 813 485 477 835