Keras Starter

From: https://www.kaggle.com/ateplyuk/keras-starter

Author: Alexander Teplyuk

Score: 0.488

Simple example of transfer learning from pretrained model using Keras.

  • Loss: Focal loss
  • Metrics: f2_score
In [1]:
import os
import cv2
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import json
from keras.models import Sequential, Model
from keras.layers import Dense, Flatten, Activation, Dropout, GlobalAveragePooling2D
from keras.preprocessing.image import ImageDataGenerator
from keras import optimizers, applications
from keras.callbacks import ModelCheckpoint, LearningRateScheduler, TensorBoard, EarlyStopping
from keras import backend as K 
Using TensorFlow backend.
In [2]:
train_df = pd.read_csv("../input/imet-2019-fgvc6/train.csv")
train_df["attribute_ids"]=train_df["attribute_ids"].apply(lambda x:x.split(" "))
train_df["id"]=train_df["id"].apply(lambda x:x+".png")
train_df.head()
Out[2]:
id attribute_ids
0 1000483014d91860.png [147, 616, 813]
1 1000fe2e667721fe.png [51, 616, 734, 813]
2 1001614cb89646ee.png [776]
3 10041eb49b297c08.png [51, 671, 698, 813, 1092]
4 100501c227f8beea.png [13, 404, 492, 903, 1093]
In [3]:
label_df = pd.read_csv("../input/imet-2019-fgvc6/labels.csv")
print(label_df.shape)
label_df.head()
(1103, 2)
Out[3]:
attribute_id attribute_name
0 0 culture::abruzzi
1 1 culture::achaemenid
2 2 culture::aegean
3 3 culture::afghan
4 4 culture::after british
In [4]:
# Example of images with tags

i = 1
plt.figure(figsize=[30,30])
for img_name in os.listdir("../input/imet-2019-fgvc6/train/")[5:10]:   
    img = cv2.imread("../input/imet-2019-fgvc6/train/%s" % img_name)[...,[2, 1, 0]]
    plt.subplot(5, 1, i)
    plt.imshow(img)
    ids = train_df[train_df["id"] == img_name]["attribute_ids"]
    title_val = []
    for tag_id in ids.values[0]:
        att_name = label_df[label_df['attribute_id'].astype(str) == tag_id]['attribute_name'].values[0]
        title_val.append(att_name)
    plt.title(title_val)
    i += 1
    
plt.show()
In [5]:
nb_classes = 1103
batch_size = 300
img_size = 75
nb_epochs = 25
In [6]:
lbls = list(map(str, range(nb_classes)))
In [7]:
%%time

train_datagen=ImageDataGenerator(
    rescale=1./255, 
    validation_split=0.25,
    horizontal_flip = True,    
    zoom_range = 0.3,
    width_shift_range = 0.3,
    height_shift_range=0.3
    )

train_generator=train_datagen.flow_from_dataframe(
    dataframe=train_df,
    directory="../input/imet-2019-fgvc6/train",
    x_col="id",
    y_col="attribute_ids",
    batch_size=batch_size,
    shuffle=True,
    class_mode="categorical",
    classes=lbls,
    target_size=(img_size,img_size),
    subset='training')

valid_generator=train_datagen.flow_from_dataframe(
    dataframe=train_df,
    directory="../input/imet-2019-fgvc6/train",
    x_col="id",
    y_col="attribute_ids",
    batch_size=batch_size,
    shuffle=True,
    class_mode="categorical",    
    classes=lbls,
    target_size=(img_size,img_size),
    subset='validation')
Found 81928 images belonging to 1103 classes.
Found 27309 images belonging to 1103 classes.
CPU times: user 4.74 s, sys: 5.39 s, total: 10.1 s
Wall time: 1min 2s
In [8]:
# Loss

gamma = 2.0
epsilon = K.epsilon()
def focal_loss(y_true, y_pred):
    pt = y_pred * y_true + (1-y_pred) * (1-y_true)
    pt = K.clip(pt, epsilon, 1-epsilon)
    CE = -K.log(pt)
    FL = K.pow(1-pt, gamma) * CE
    loss = K.sum(FL, axis=1)
    return loss
In [9]:
# Metric

def f2_score(y_true, y_pred):
    beta = 2
    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)), axis=1)
    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)), axis=1)
    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)), axis=1)
    
    precision = true_positives / (predicted_positives + K.epsilon())
    recall = true_positives / (possible_positives + K.epsilon())
    
    return K.mean(((1+beta**2)*precision*recall) / ((beta**2)*precision+recall+K.epsilon()))
In [10]:
model = applications.InceptionResNetV2(weights=None, 
                          include_top=False, 
                          input_shape=(img_size, img_size, 3))
model.load_weights('../input/inceptionresnetv2/inception_resnet_v2_weights_tf_dim_ordering_tf_kernels_notop.h5')
WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
In [11]:
model.trainable = False
In [12]:
# Freeze some layers
# for layer in model.layers[:-4]:
#     layer.trainable = False
In [13]:
#Adding custom layers 
x = model.output
x = Flatten()(x)
x = Dense(1024, activation="relu")(x)
x = Dropout(0.5)(x)
predictions = Dense(nb_classes, activation="softmax")(x)
model_final = Model(input = model.input, output = predictions)

model_final.compile(optimizers.rmsprop(lr=0.001, decay=1e-6),loss=focal_loss,metrics=[f2_score])
WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.
Instructions for updating:
Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.
/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:7: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=Tensor("in..., outputs=Tensor("de...)`
  import sys
In [14]:
# model_final.summary()
In [15]:
# Callbacks

checkpoint = ModelCheckpoint("model_1.h5", monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False, mode='auto', period=1)
early = EarlyStopping(monitor='val_loss', min_delta=0, patience=5, verbose=1, mode='auto')
In [16]:
%%time
history = model_final.fit_generator(generator=train_generator,                   
                                    steps_per_epoch=500,
                                    validation_data=valid_generator,                    
                                    validation_steps=200,
                                    epochs=nb_epochs,
                                    callbacks = [checkpoint, early],
                                    max_queue_size=16,
                                    workers=2,
                                    use_multiprocessing=True,
                                    verbose=0)
WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
/opt/conda/lib/python3.6/site-packages/keras/engine/training_generator.py:47: UserWarning: Using a generator with `use_multiprocessing=True` and multiple workers may duplicate your data. Please consider using the`keras.utils.Sequence class.
  UserWarning('Using a generator with `use_multiprocessing=True`'
Epoch 00001: val_loss improved from inf to 20.84247, saving model to model_1.h5

Epoch 00002: val_loss improved from 20.84247 to 13.61142, saving model to model_1.h5

Epoch 00003: val_loss improved from 13.61142 to 12.66795, saving model to model_1.h5

Epoch 00004: val_loss did not improve from 12.66795

Epoch 00005: val_loss improved from 12.66795 to 12.46695, saving model to model_1.h5

Epoch 00006: val_loss improved from 12.46695 to 12.19122, saving model to model_1.h5

Epoch 00007: val_loss did not improve from 12.19122

Epoch 00008: val_loss did not improve from 12.19122

Epoch 00009: val_loss did not improve from 12.19122

Epoch 00010: val_loss improved from 12.19122 to 11.95743, saving model to model_1.h5

Epoch 00011: val_loss improved from 11.95743 to 11.83258, saving model to model_1.h5

Epoch 00012: val_loss did not improve from 11.83258

Epoch 00013: val_loss did not improve from 11.83258

Epoch 00014: val_loss did not improve from 11.83258

Epoch 00015: val_loss did not improve from 11.83258

Epoch 00016: val_loss did not improve from 11.83258
Epoch 00016: early stopping
CPU times: user 49min 59s, sys: 15min 30s, total: 1h 5min 30s
Wall time: 4h 54min 10s
In [17]:
with open('history.json', 'w') as f:
    json.dump(history.history, f)

history_df = pd.DataFrame(history.history)
history_df[['loss', 'val_loss']].plot()
history_df[['f2_score', 'val_f2_score']].plot()
Out[17]:
<matplotlib.axes._subplots.AxesSubplot at 0x7f8b781f9cf8>
In [18]:
sam_sub_df = pd.read_csv('../input/imet-2019-fgvc6/sample_submission.csv')
sam_sub_df["id"]=sam_sub_df["id"].apply(lambda x:x+".png")
print(sam_sub_df.shape)
sam_sub_df.head()
(7443, 2)
Out[18]:
id attribute_ids
0 10023b2cc4ed5f68.png 0 1 2
1 100fbe75ed8fd887.png 0 1 2
2 101b627524a04f19.png 0 1 2
3 10234480c41284c6.png 0 1 2
4 1023b0e2636dcea8.png 0 1 2
In [19]:
%%time
test_datagen = ImageDataGenerator(rescale=1./255)
test_generator = test_datagen.flow_from_dataframe(  
        dataframe=sam_sub_df,
        directory = "../input/imet-2019-fgvc6/test",    
        x_col="id",
        target_size = (img_size,img_size),
        batch_size = 1,
        shuffle = False,
        class_mode = None
        )
Found 7443 images.
CPU times: user 148 ms, sys: 212 ms, total: 360 ms
Wall time: 2.01 s
In [20]:
%%time
test_generator.reset()
predict = model_final.predict_generator(test_generator, steps = len(test_generator.filenames))
CPU times: user 6min 2s, sys: 7.7 s, total: 6min 10s
Wall time: 4min 57s
In [21]:
len(predict)
Out[21]:
7443
In [22]:
%%time
import operator
predicted_class_indices_3=[]
for i in range(len(predict)):         
    d = {}
    for index, value in enumerate(predict[i]):               
        if value > 0.03:            
            d[index] = value 
    sorted_d = sorted(d.items(), key=operator.itemgetter(1), reverse=True)
    
    # Take only first 10 items
    predicted_class_indices_3.append([i[0] for i in sorted_d[:10]])
CPU times: user 12.6 s, sys: 0 ns, total: 12.6 s
Wall time: 12.6 s
In [23]:
%%time
predictions_3=[]

for i in range(len(predicted_class_indices_3)):
    labels = (train_generator.class_indices)
    labels = dict((v,k) for k,v in labels.items())
    predictions = [labels[k] for k in predicted_class_indices_3[i]]
    predictions_3.append(predictions)
CPU times: user 1.3 s, sys: 0 ns, total: 1.3 s
Wall time: 1.3 s
In [24]:
predict_3 = []
for i in range(len(predictions_3)):
    str3 = " ".join(predictions_3[i])
    predict_3.append(str3)
In [25]:
filenames=test_generator.filenames
results=pd.DataFrame({"id":filenames,
                      "attribute_ids":predict_3})
results['id'] = results['id'].map(lambda x: str(x)[:-4])
results.to_csv("submission.csv",index=False)
In [26]:
results.head()
Out[26]:
id attribute_ids
0 10023b2cc4ed5f68 587 1059 369 766 1020
1 100fbe75ed8fd887 1039 231 226 1059 121
2 101b627524a04f19 498 728 180 497 182 482
3 10234480c41284c6 1046 111 813 147 776 1092 189 259
4 1023b0e2636dcea8 584 954 156 227 671 813 147 1046 1092