inaturalist xception 512

From: https://www.kaggle.com/cedriclacrambe/inaturalist-xception-512

Author: Cédric Lacrambe

In [1]:
# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load in 
import  datetime
date_depart=datetime.datetime.now()
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import weakref
import warnings

warnings.simplefilter("ignore")
import logging
logging.basicConfig(filename='python.log',level=logging.DEBUG)
logging.captureWarnings(True)
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
# tell the handler to use this format
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)

from functools import partial
# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory
import cv2
import matplotlib.pyplot as plt
import imageio
import imgaug
from imgaug import augmenters as iaa
from imgaug import parameters as iap
import keras
from keras.models import Sequential, Model
from keras.layers import Dense, Flatten, Activation, Dropout, GlobalAveragePooling2D
from keras.preprocessing.image import ImageDataGenerator
from keras import optimizers, applications
from keras.callbacks import ModelCheckpoint, LearningRateScheduler, TensorBoard, EarlyStopping
from keras import backend as K 
from concurrent import futures
import os
import json
print(os.listdir("../input"))

# Any results you write to the current directory are saved as output.
Using TensorFlow backend.
['inaturalist-2019-fgvc6', 'xception', 'inaturalist-xception-512']
In [2]:
#!ls -Rlh ../input
In [3]:
duree_max=datetime.timedelta(hours=7,minutes=30)

fichier_modele_base="inaturalist" 
train_batch_size=64
full_train_batch_size=4
val_batch_size=4
epochs=int(1e8)

load_keras_weights=False
dmax=1024
dmin=600
dcrop=512
date_limite= date_depart+duree_max
In [4]:
ann_file = '../input/inaturalist-2019-fgvc6/train2019.json'
with open(ann_file) as data_file:
        train_anns = json.load(data_file)
def get_file_list(ann_file):
    with open(ann_file) as data_file:
            train_anns = json.load(data_file)

    train_anns_df = pd.DataFrame(train_anns['annotations'])[['image_id','category_id']]
    train_img_df = pd.DataFrame(train_anns['images'])[['id', 'file_name']].rename(columns={'id':'image_id'})
    train_anns_df [train_anns_df.image_id.duplicated()]

    df_train=pd.merge(train_img_df,train_anns_df)[['category_id', 'file_name']]
    return  df_train


df_train=get_file_list(ann_file)
df_train
Out[4]:
category_id file_name
0 400 train_val2019/Plants/400/d1322d13ccd856eb4236c...
1 570 train_val2019/Plants/570/15edbc1e2ef000d8ace48...
2 167 train_val2019/Reptiles/167/c87a32e8927cbf4f06d...
3 254 train_val2019/Birds/254/9fcdd1d37e96d8fd94dfdc...
4 739 train_val2019/Plants/739/ffa06f951e99de9d220ae...
5 246 train_val2019/Birds/246/ef5109f4c0bd04ca509fad...
6 25 train_val2019/Insects/25/5ee1691712dffd9e3a178...
7 386 train_val2019/Plants/386/3af7b0ad2e2bc118e38c5...
8 515 train_val2019/Plants/515/4bfd750cbb32bb0e302ca...
9 259 train_val2019/Birds/259/353bd26bf9e5e1606a9189...
10 824 train_val2019/Plants/824/2543714c2f1a8611f1a78...
11 234 train_val2019/Birds/234/dc64a999cacea6eb5ed512...
12 363 train_val2019/Plants/363/aec09afe1abc6aa7dc70c...
13 235 train_val2019/Birds/235/73d440f2157a161fec3e77...
14 862 train_val2019/Plants/862/e4d28d6347aa0d38322a6...
15 731 train_val2019/Plants/731/032b0860dcd8ae04aaab4...
16 325 train_val2019/Birds/325/c1db84df7d22e0933df6b6...
17 213 train_val2019/Birds/213/091a60ad7d3cbec3141eea...
18 149 train_val2019/Insects/149/2cd40eee1996f0eb3187...
19 131 train_val2019/Insects/131/6e84516b34cdc45be2a3...
20 167 train_val2019/Reptiles/167/e21448daf063c25cb56...
21 151 train_val2019/Insects/151/c84bc5a5187096098f4f...
22 751 train_val2019/Plants/751/c3fb975e1120f46671632...
23 270 train_val2019/Birds/270/8d00e0ba26503ec41aa884...
24 205 train_val2019/Birds/205/430ba5e10dc2a2888f8b14...
25 415 train_val2019/Plants/415/c49a1cd3a1d83a15e9274...
26 149 train_val2019/Insects/149/e0587065186b2bd34478...
27 739 train_val2019/Plants/739/16494b1e8cdcc0cbf5c13...
28 8 train_val2019/Fungi/8/ceea3c6d5ff90364486f5683...
29 922 train_val2019/Plants/922/9635afacfa8e46e0390c2...
... ... ...
265183 604 train_val2019/Plants/604/32958abfc2eb70b849356...
265184 236 train_val2019/Birds/236/e2197c5fa669685dc45221...
265185 660 train_val2019/Plants/660/6300ce1be902e9c5ca97e...
265186 353 train_val2019/Plants/353/39882460221ea4acf9879...
265187 387 train_val2019/Plants/387/e9060a2a376ef9d53d55e...
265188 907 train_val2019/Plants/907/45503a52d6ff851ee2aac...
265189 968 train_val2019/Plants/968/0ee365c29cbf8e328ccb1...
265190 188 train_val2019/Reptiles/188/128f298609594ba84cc...
265191 153 train_val2019/Amphibians/153/f1e8b2d3b668b9922...
265192 970 train_val2019/Plants/970/8fd903f36f2132605d3bb...
265193 350 train_val2019/Plants/350/c60f16cb442d9f40e11eb...
265194 712 train_val2019/Plants/712/649689c72339cd76f0220...
265195 526 train_val2019/Plants/526/feb980e3cad11fbf56940...
265196 233 train_val2019/Birds/233/c06b3e72c9796fc266969c...
265197 328 train_val2019/Plants/328/65bfb2a6fe3dceb3e0b02...
265198 211 train_val2019/Birds/211/f1f83215f9cdbc29064b9e...
265199 268 train_val2019/Birds/268/6f303db38e86c4dd5ba943...
265200 321 train_val2019/Birds/321/540a4789c221ca78be9a30...
265201 273 train_val2019/Birds/273/335c36a57fbd52e6071c4e...
265202 477 train_val2019/Plants/477/04234ec77dd9d48512a6d...
265203 420 train_val2019/Plants/420/5802a39c2f868ffa4ad6e...
265204 77 train_val2019/Insects/77/41f0485a9fc909328b8a5...
265205 1001 train_val2019/Plants/1001/c1813d4501453affaeaf...
265206 375 train_val2019/Plants/375/d4b9b81094127add57afe...
265207 532 train_val2019/Plants/532/19b8c68c716998290f242...
265208 223 train_val2019/Birds/223/ebf4f99d55fb0071c49e08...
265209 188 train_val2019/Reptiles/188/cb0600062644540aaf2...
265210 70 train_val2019/Insects/70/bfd566f9ae5177f5f2b52...
265211 587 train_val2019/Plants/587/59c3e60d021606bf352de...
265212 287 train_val2019/Birds/287/99da04ba30e67391f9a35b...

265213 rows × 2 columns

In [5]:
df_val=get_file_list('../input/inaturalist-2019-fgvc6/val2019.json')
df_val
Out[5]:
category_id file_name
0 644 train_val2019/Plants/644/716a69838526f3ada3b2f...
1 597 train_val2019/Plants/597/0942cc64d2e759c5ee059...
2 883 train_val2019/Plants/883/acfdbfd9fa675f1c84558...
3 300 train_val2019/Birds/300/5f3194ff536c7dd31d80b7...
4 881 train_val2019/Plants/881/76acaf0b2841f91982d21...
5 771 train_val2019/Plants/771/5d190fd90da893988a3c9...
6 607 train_val2019/Plants/607/8fb0ddfe92dadbf9c5753...
7 698 train_val2019/Plants/698/6d71521a64d1e2fe8bb34...
8 714 train_val2019/Plants/714/09508f2bf937d21d63297...
9 316 train_val2019/Birds/316/21c700bc90523485af6730...
10 726 train_val2019/Plants/726/847b2ebf8efc1a528c2d3...
11 27 train_val2019/Insects/27/e5d141185a3a50b544d15...
12 640 train_val2019/Plants/640/cbe5ff1a159b614ae8677...
13 756 train_val2019/Plants/756/a99c33a5f954ffecd7340...
14 73 train_val2019/Insects/73/2fc6b41247af765af9984...
15 594 train_val2019/Plants/594/d98519788816a0f61bd15...
16 179 train_val2019/Reptiles/179/914aab5662e77a128c7...
17 440 train_val2019/Plants/440/02f1f67068df3d8855467...
18 859 train_val2019/Plants/859/23ba5d0baf198007ec097...
19 5 train_val2019/Fungi/5/3308225981fc58e6eb6a7889...
20 401 train_val2019/Plants/401/6c73829cb13a02a69123e...
21 928 train_val2019/Plants/928/d65f36eaf70136e1d4c2e...
22 216 train_val2019/Birds/216/f5902f2a9566038b67ff72...
23 200 train_val2019/Reptiles/200/ef2471af2f01a252250...
24 983 train_val2019/Plants/983/5b1e6397e48bec711b3a3...
25 350 train_val2019/Plants/350/8ebad1a2af78183b35bac...
26 223 train_val2019/Birds/223/e951479ab2f49714025c1e...
27 438 train_val2019/Plants/438/f94013bba01de6a654f81...
28 834 train_val2019/Plants/834/4c3d81073f661f93c20c3...
29 248 train_val2019/Birds/248/0a2da2111a989327d9935b...
... ... ...
3000 681 train_val2019/Plants/681/3458385697acabf822006...
3001 830 train_val2019/Plants/830/9a30a84d57d2aa127298e...
3002 60 train_val2019/Insects/60/cb0032132cbad87f49115...
3003 784 train_val2019/Plants/784/0cadfce23ea5c5aaff90b...
3004 873 train_val2019/Plants/873/b6a4bebe95e25647b50fd...
3005 868 train_val2019/Plants/868/7e6410f5ea26657666206...
3006 704 train_val2019/Plants/704/e226a1d3713feaafc0443...
3007 617 train_val2019/Plants/617/dbea4253dbcfb3b06b3c1...
3008 631 train_val2019/Plants/631/191450a32f3105071caf2...
3009 548 train_val2019/Plants/548/edd75e1d3fec72691b046...
3010 424 train_val2019/Plants/424/047f1de7a61605cb15b5a...
3011 14 train_val2019/Insects/14/d1d2163a1b64882cd2a7c...
3012 976 train_val2019/Plants/976/963055c5e9726e06ae73c...
3013 321 train_val2019/Birds/321/98713839fa00a49b75dc5e...
3014 391 train_val2019/Plants/391/7628608962c27c640c38d...
3015 862 train_val2019/Plants/862/f9969d436af626938b9dc...
3016 809 train_val2019/Plants/809/fe521339f28d17599928c...
3017 813 train_val2019/Plants/813/275660d70fe22e5ddc0c0...
3018 219 train_val2019/Birds/219/99e8afe389a7b46fefdb86...
3019 292 train_val2019/Birds/292/5a1af52f0434e3996f7a5a...
3020 155 train_val2019/Amphibians/155/009611cb3d44f7a9c...
3021 772 train_val2019/Plants/772/d83e9fd8c2e3493df4e49...
3022 968 train_val2019/Plants/968/59d8da88969576e7bae10...
3023 893 train_val2019/Plants/893/627c72786a18242784b7f...
3024 157 train_val2019/Amphibians/157/6d3b4f10fa385dd81...
3025 468 train_val2019/Plants/468/3260e4346ba127a28b564...
3026 1006 train_val2019/Plants/1006/3b33c0e1fb866c452f01...
3027 472 train_val2019/Plants/472/44a19d7731dccd92e4c0e...
3028 192 train_val2019/Reptiles/192/c5f142b44904508c866...
3029 835 train_val2019/Plants/835/a6f21e1829ff47c54d474...

3030 rows × 2 columns

In [6]:
train_anns.keys()
Out[6]:
dict_keys(['info', 'images', 'licenses', 'annotations', 'categories'])
In [7]:
len(df_train.category_id.unique())
Out[7]:
1010
In [8]:
df_valid=get_file_list('../input/inaturalist-2019-fgvc6/train2019.json')
df_valid
Out[8]:
category_id file_name
0 400 train_val2019/Plants/400/d1322d13ccd856eb4236c...
1 570 train_val2019/Plants/570/15edbc1e2ef000d8ace48...
2 167 train_val2019/Reptiles/167/c87a32e8927cbf4f06d...
3 254 train_val2019/Birds/254/9fcdd1d37e96d8fd94dfdc...
4 739 train_val2019/Plants/739/ffa06f951e99de9d220ae...
5 246 train_val2019/Birds/246/ef5109f4c0bd04ca509fad...
6 25 train_val2019/Insects/25/5ee1691712dffd9e3a178...
7 386 train_val2019/Plants/386/3af7b0ad2e2bc118e38c5...
8 515 train_val2019/Plants/515/4bfd750cbb32bb0e302ca...
9 259 train_val2019/Birds/259/353bd26bf9e5e1606a9189...
10 824 train_val2019/Plants/824/2543714c2f1a8611f1a78...
11 234 train_val2019/Birds/234/dc64a999cacea6eb5ed512...
12 363 train_val2019/Plants/363/aec09afe1abc6aa7dc70c...
13 235 train_val2019/Birds/235/73d440f2157a161fec3e77...
14 862 train_val2019/Plants/862/e4d28d6347aa0d38322a6...
15 731 train_val2019/Plants/731/032b0860dcd8ae04aaab4...
16 325 train_val2019/Birds/325/c1db84df7d22e0933df6b6...
17 213 train_val2019/Birds/213/091a60ad7d3cbec3141eea...
18 149 train_val2019/Insects/149/2cd40eee1996f0eb3187...
19 131 train_val2019/Insects/131/6e84516b34cdc45be2a3...
20 167 train_val2019/Reptiles/167/e21448daf063c25cb56...
21 151 train_val2019/Insects/151/c84bc5a5187096098f4f...
22 751 train_val2019/Plants/751/c3fb975e1120f46671632...
23 270 train_val2019/Birds/270/8d00e0ba26503ec41aa884...
24 205 train_val2019/Birds/205/430ba5e10dc2a2888f8b14...
25 415 train_val2019/Plants/415/c49a1cd3a1d83a15e9274...
26 149 train_val2019/Insects/149/e0587065186b2bd34478...
27 739 train_val2019/Plants/739/16494b1e8cdcc0cbf5c13...
28 8 train_val2019/Fungi/8/ceea3c6d5ff90364486f5683...
29 922 train_val2019/Plants/922/9635afacfa8e46e0390c2...
... ... ...
265183 604 train_val2019/Plants/604/32958abfc2eb70b849356...
265184 236 train_val2019/Birds/236/e2197c5fa669685dc45221...
265185 660 train_val2019/Plants/660/6300ce1be902e9c5ca97e...
265186 353 train_val2019/Plants/353/39882460221ea4acf9879...
265187 387 train_val2019/Plants/387/e9060a2a376ef9d53d55e...
265188 907 train_val2019/Plants/907/45503a52d6ff851ee2aac...
265189 968 train_val2019/Plants/968/0ee365c29cbf8e328ccb1...
265190 188 train_val2019/Reptiles/188/128f298609594ba84cc...
265191 153 train_val2019/Amphibians/153/f1e8b2d3b668b9922...
265192 970 train_val2019/Plants/970/8fd903f36f2132605d3bb...
265193 350 train_val2019/Plants/350/c60f16cb442d9f40e11eb...
265194 712 train_val2019/Plants/712/649689c72339cd76f0220...
265195 526 train_val2019/Plants/526/feb980e3cad11fbf56940...
265196 233 train_val2019/Birds/233/c06b3e72c9796fc266969c...
265197 328 train_val2019/Plants/328/65bfb2a6fe3dceb3e0b02...
265198 211 train_val2019/Birds/211/f1f83215f9cdbc29064b9e...
265199 268 train_val2019/Birds/268/6f303db38e86c4dd5ba943...
265200 321 train_val2019/Birds/321/540a4789c221ca78be9a30...
265201 273 train_val2019/Birds/273/335c36a57fbd52e6071c4e...
265202 477 train_val2019/Plants/477/04234ec77dd9d48512a6d...
265203 420 train_val2019/Plants/420/5802a39c2f868ffa4ad6e...
265204 77 train_val2019/Insects/77/41f0485a9fc909328b8a5...
265205 1001 train_val2019/Plants/1001/c1813d4501453affaeaf...
265206 375 train_val2019/Plants/375/d4b9b81094127add57afe...
265207 532 train_val2019/Plants/532/19b8c68c716998290f242...
265208 223 train_val2019/Birds/223/ebf4f99d55fb0071c49e08...
265209 188 train_val2019/Reptiles/188/cb0600062644540aaf2...
265210 70 train_val2019/Insects/70/bfd566f9ae5177f5f2b52...
265211 587 train_val2019/Plants/587/59c3e60d021606bf352de...
265212 287 train_val2019/Birds/287/99da04ba30e67391f9a35b...

265213 rows × 2 columns

In [9]:
classes=len(df_valid.category_id.unique())
print("random accuracy:",1/classes)
classes
random accuracy: 0.0009900990099009901
Out[9]:
1010
In [10]:
df_train["catstr"]=df_train.category_id.astype("str")
In [11]:
p=next(df_train.sample(n=5).itertuples())
p.category_id
p.file_name
Out[11]:
'train_val2019/Birds/246/ee8c33c450ac2243493896decff953be.jpg'
In [12]:
plt.figure(figsize=(25,25))
for n,tu in enumerate(df_train.sample(n=5).itertuples()):
    cat=tu.category_id
    im=tu.file_name
    plt.subplot(1,5,n+1)
    im=os.path.join("../input/inaturalist-2019-fgvc6/train_val2019/",im)
    plt.axis("off")
    plt.title(cat)
    plt.imshow(imageio.imread(im)) 
In [13]:
augmenters=[iaa.Sometimes(0.5,             
                                [iaa.Affine(scale=(0.99,1.05),
                                 translate_percent=(0,0.05), 
                                 rotate=iap.Normal(0,3),
                                 shear=iap.Normal(0,3),
                                 order=3)]),
                      iaa.Sometimes(0.3,[iaa.PiecewiseAffine(scale=(0,0.02))]),
                 
                                                        
   
                    iaa.Sometimes(0.1,
                                    [iaa.GaussianBlur(sigma=(0, 0.5)) ]),
                    iaa.Sometimes(0.1,
                                        [iaa.AverageBlur(k=(1, 2))]),
                    iaa.AdditiveGaussianNoise(scale=(0, 0.05*255)),
                    iaa.AddElementwise((-5, 5)),
                     
                    iaa.Sometimes(0.1,
                                    [iaa.Superpixels(p_replace=(0.05, 0.8), n_segments=(16, 128))]
                                 ),
                    
            
            
            
                    iaa.Sometimes(0.3,
                                    [iaa.ElasticTransformation(alpha=(0, 5.0), sigma=(0.1,0.6) )]
                                 ),
                    
                    iaa.SaltAndPepper(p=(0.005,0.1)),
                                      iaa.Sometimes(0.4,
                                        [iaa.CoarseDropout(p=(0, 0.3),size_percent=(0.02, 0.5))]),

                        
                      
]

augmenters=[iaa.Sometimes(0.8,[iaa.Sequential(augmenters)])]
In [14]:
all(os.path.isfile(os.path.join("../input/inaturalist-2019-fgvc6/train_val2019/",f)) for f in df_train.sample(n=6000).file_name)
Out[14]:
True
In [15]:
tu=next(df_train.itertuples())
tu.file_name
tu.category_id

df=df_train.copy()
df["req"]=None
In [16]:
def  imgaug_batch_gen(df,batch=16,executor=None,dmax=1024,dmin=512):
    if executor is None:
        executor=futures.ThreadPoolExecutor(max_workers=2)
    prefetch=int(batch*2.1+1) 
    df_len=len(df)
    def load_resize(f,dmax=dmax,dmin=dmin):
        img=imageio.imread(f,pilmode="RGB")
        resmin_img=np.min( img.shape[:2] )
        resmin=np.clip(resmin_img,dmin,dmax)
        r=resmin_img/resmin
            
        if r>1.2 or r<1.0:
            img=cv2.resize(img,None,fx=1.05/r, fy=1.05/r, interpolation = cv2.INTER_CUBIC)
        return img


    while True:
        
        df=df[["category_id","file_name"]].sample(frac=1).reset_index(drop=True)
        df["req"]=None
        i=0
        while i <(df_len-batch):
            df[df.req.notnull()].loc[:i]=None
            for j in range(prefetch):
                try:
                    if df.loc[j+i,"req"] is None:
                        f=os.path.join("../input/inaturalist-2019-fgvc6/train_val2019/",df.loc[j+i,"file_name"])
                        df.loc[j+i,"req"]=executor.submit(load_resize ,f)   
                except KeyError:
                    logging.exception("imgaug_batch_gen")

            df_batch=df.loc[i:i+batch-1]
            imgs=[req.result() for req in  df_batch.req]
            resmin=np.min(np.array([im.shape[:2]  for im in imgs]))
            resmin=np.clip(resmin,dmin,dmax)
            for j in range(batch):
                x,y,_=imgs[j].shape
                img_resmin=min(x,y)
                r=img_resmin/resmin
            
                if r>1.2 or r<1.0:
                    imgs[j]=cv2.resize(imgs[j],None,fx=1.05/r, fy=1.05/r, interpolation = cv2.INTER_CUBIC)
           
            categories=df_batch.category_id.values.astype("int32")                                        
            
            yield imgaug.imgaug.Batch(images=imgs,data=categories) 
            df.loc[i:i+batch-1,"req"]=None
            i+=batch
   


def  batch_gen(df,batch=16,augmenters=[],executor=None,dmax=1024,dmin=512,dcrop=512):
    if executor is None:
        executor=futures.ThreadPoolExecutor(max_workers=2)
    
    dmin=max(dmin,dcrop)
    dmax=max(dmin,dmax)
    gen =imgaug_batch_gen(df,batch=batch,executor=executor,dmax=dmax,dmin=dmin)
    aug=iaa.Sequential(augmenters+[iaa.CropToFixedSize(dmin,dmin),iaa.PadToFixedSize(dmin,dmin)])
    #aug_pool=aug.pool(processes=-1,maxtasksperchild=8)
    #gen=aug_pool.imap_batches_unordered(gen, chunksize=1)
    
    def aug_closure_gen(gen=gen):
        b=next(gen)
        fut=executor.submit(aug.augment_batches,[b],background=False)
        for b in gen:
            imgs=list(fut.result())[0].images_aug
            fut=executor.submit(aug.augment_batches,[b],background=False)                
            #b=list(aug.augment_batches([b],background=False))[0]

            
            imgs=[im[None,...] for im in imgs]
            X=np.concatenate(imgs).astype("float32")/256
            Y=b.data[...,None]      
            yield X,Y
   
    aug_closure=aug_closure_gen(gen)

    return aug_closure
            
        
        
    
    
                                                         
                                               
aug=iaa.Sequential(augmenters)
#aug_pool=aug.pool(processes=None, maxtasksperchild=None)

gen=batch_gen(df_train,batch=16,augmenters=augmenters)
X,Y=next(gen)
del gen
b_len=X.shape[0]
cols=4
rows=b_len//cols
if b_len%cols!=0:
    rows=rows+1
    
plt.figure(figsize=(25,25))
X.shape[0]
for n in range(b_len):
    
    cat=Y[n][0]

    plt.subplot(rows,cols,n+1)
    
    plt.axis("off")
    plt.title(cat)
    plt.imshow(X[n])      
    
    
In [17]:
pretrained_model=keras.applications.Xception(include_top=False, weights=None,pooling=None)
pretrained_model.load_weights("../input/xception/xception_weights_tf_dim_ordering_tf_kernels_notop.h5")
fichier_modele=f"{fichier_modele_base}_{pretrained_model.name}.h5" 
fichier_modele
tensorflow  : WARNING  From /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
Out[17]:
'inaturalist_xception.h5'
In [18]:
def mean_anwer_guess(y_true, y_pred):
    n=K.get_variable_shape(y_pred)[1]
    y_pred=K.cast(y_pred,"int64")
    preds=K.gather(y_true,y_pred)
    return K.mean(preds)


def mean_prandom_ratio(y_true, y_pred):
    n=K.get_variable_shape(y_pred)[1]
    y_pred=K.cast(y_pred,"int64")
    preds=K.gather(y_true,y_pred)
    return K.mean(preds*n)
import functools
def get_sparse_topn__categorical_accuracy(k):
    func=functools.partial(keras.metrics.sparse_top_k_categorical_accuracy,k=k)
    func.__name__=f"sparse_top_{k}_categorical_accuracy"
    return func
In [19]:
out_regulariser=keras.regularizers.l1_l2(l1=0.01, l2=0.05)

image_input=keras.Input(shape=(None,None,3), name="image_input", dtype="float32")
bottleneck1=pretrained_model(image_input)
bottleneck=keras.layers.Conv2D(filters=600,
                               kernel_size=3,padding="same", 
                               kernel_initializer=keras.initializers.Orthogonal(),
                               activation="selu",
                               activity_regularizer=keras.regularizers.l1_l2(l1=0.01, l2=0.05),
                               strides=2
                              
                              )(bottleneck1)


pool1=(keras.layers.GlobalMaxPool2D()(bottleneck1))

pool=(keras.layers.GlobalMaxPool2D()(bottleneck))
pool=keras.layers.Concatenate()([pool,pool1])
pool=keras.layers.AlphaDropout(0.3)(pool)
pre_out=keras.layers.Dense(1200 ,
                           name="pre_out",
                            activation="selu",
                           kernel_regularizer=out_regulariser
                          )(pool)
out=keras.layers.Dense(classes,
                       activation="softmax"
                       ,name="out",
                       kernel_initializer=keras.initializers.Orthogonal(),
                       kernel_regularizer=out_regulariser)(pre_out)

model=keras.Model(inputs=image_input,outputs=out)
pretrained_model.trainable=False
optimizer=keras.optimizers.Adam(clipnorm=5. , clipvalue=5.,amsgrad=False,lr=0.0005)
model.compile(optimizer,
                            loss="sparse_categorical_crossentropy",
              metrics=["sparse_categorical_accuracy","sparse_categorical_crossentropy",mean_anwer_guess,get_sparse_topn__categorical_accuracy(2),
                       get_sparse_topn__categorical_accuracy(5),get_sparse_topn__categorical_accuracy(10)]
             )
model.summary()
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
image_input (InputLayer)        (None, None, None, 3 0                                            
__________________________________________________________________________________________________
xception (Model)                (None, None, None, 2 20861480    image_input[0][0]                
__________________________________________________________________________________________________
conv2d_5 (Conv2D)               (None, None, None, 6 11059800    xception[1][0]                   
__________________________________________________________________________________________________
global_max_pooling2d_2 (GlobalM (None, 600)          0           conv2d_5[0][0]                   
__________________________________________________________________________________________________
global_max_pooling2d_1 (GlobalM (None, 2048)         0           xception[1][0]                   
__________________________________________________________________________________________________
concatenate_1 (Concatenate)     (None, 2648)         0           global_max_pooling2d_2[0][0]     
                                                                 global_max_pooling2d_1[0][0]     
__________________________________________________________________________________________________
alpha_dropout_1 (AlphaDropout)  (None, 2648)         0           concatenate_1[0][0]              
__________________________________________________________________________________________________
pre_out (Dense)                 (None, 1200)         3178800     alpha_dropout_1[0][0]            
__________________________________________________________________________________________________
out (Dense)                     (None, 1010)         1213010     pre_out[0][0]                    
==================================================================================================
Total params: 36,313,090
Trainable params: 15,451,610
Non-trainable params: 20,861,480
__________________________________________________________________________________________________
In [20]:
class termination_date(keras.callbacks.Callback ):
    def __init__(self,end_date):
        self.end_date=end_date
    def on_epoch_end(self, batch, logs=None):
        if datetime.datetime.now()>self.end_date:
            self.model.stop_training = True
            logging.info("end date")
            
            
class logcallback(keras.callbacks.Callback):
    def __init__(self,logger=None):
        if logger is None:
            logger=logging.getLogger('traincallback')

        self.logger=logger
    def on_train_begin(self, logs={}):
        self.logger.info("training start: %s",self.model.name)
       

    def on_batch_end(self, batch, logs={}):
        met=""
        for k,v in logs.items():
            met+=f"{k}: {str(v)} "
        self.logger.debug("batch: %s - %s",batch,met)
        
    def on_epoch_end(self, epoch, logs=None):
        met=""
        for k,v in logs.items():
            met+=f"{k}: {str(v)} "
        self.logger.info("epoch: %s - %s",epoch,met)
    def on_train_end(self, logs={}):
        self.logger.info("training end: %s",self.model.name)
        
        
        
        
In [21]:
callbacks=[
        keras.callbacks.ReduceLROnPlateau(monitor='val_sparse_categorical_accuracy',
                                          patience=10,
                                          min_delta=0.0005,
                                          factor=0.6,
                                          #min_lr=1e-6,
                                          verbose=1,
                                          cooldown=5

                                          ),
        keras.callbacks.ModelCheckpoint(monitor='val_sparse_categorical_accuracy',
                                        filepath=fichier_modele,
                                        verbose=1,
                                        save_best_only=True,
                                        period=20),
        keras.callbacks.EarlyStopping(monitor='val_sparse_categorical_accuracy',
                                      patience=20,
                                         
                                          verbose=1,
                                          restore_best_weights=True

                                          ),
         keras.callbacks.CSVLogger("train.csv", separator=',', append=True),
         termination_date(date_limite-datetime.timedelta(minutes=30)),
            keras.callbacks.BaseLogger(),
        logcallback()
    

        ]
In [22]:
if load_keras_weights:

    for fp in glob.glob(f"../input/**/{fichier_modele}",recursive=True):
        try:
            model.load_weights(fp, by_name=True, skip_mismatch=True)
            logging.info("loaded weights:",fb)
        except Exception as e:
            print(type(e),e)
            logging.exception("exception loading: %s %s",fp,e)
    if os.path.exists(fichier_modele):
        model.load_weights(fichier_modele, by_name=True, skip_mismatch=True)
        logging.info("loaded weights:",fb)
In [23]:
val_gen=batch_gen(df_val,batch=val_batch_size,dmax=dcrop,dmin=dcrop,dcrop=dcrop)
In [24]:
uptime=datetime.datetime.now()-date_depart
logging.info("pre train start %s",uptime)
root        : INFO     pre train start 0:00:41.700056
In [25]:
hist_pre=model.fit_generator(batch_gen(df_train,batch=train_batch_size,augmenters=augmenters,dmax=dmax,dmin=dmin,dcrop=dcrop),
                     steps_per_epoch=1280/train_batch_size, 
                             epochs=150,
                             verbose=1,
                     validation_data=val_gen,
                     validation_steps=300/val_batch_size,
                     callbacks=   callbacks+    [ keras.callbacks.EarlyStopping(monitor='val_sparse_categorical_accuracy',
                                      patience=20,
                                         
                                          verbose=1,
                                          restore_best_weights=True

                                          )]
                     
                    
                   )
model.save(fichier_modele)

logging.info("pre train end %s",datetime.datetime.now()-date_depart)
logging.info("remaining time %s",datetime.timedelta(hours=9)+date_depart-datetime.datetime.now())
tensorflow  : WARNING  From /opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
tensorflow  : WARNING  From /opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Deprecated in favor of operator or tf.math.divide.
traincallback: INFO     training start: model_1
Epoch 1/150
20/20 [==============================] - 336s 17s/step - loss: 72279.2942 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 7.8264 - mean_anwer_guess: 451.1500 - sparse_top_2_categorical_accuracy: 0.0031 - sparse_top_5_categorical_accuracy: 0.0070 - sparse_top_10_categorical_accuracy: 0.0109 - val_loss: 1892.6826 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 7.3714 - val_mean_anwer_guess: 473.1333 - val_sparse_top_2_categorical_accuracy: 0.0067 - val_sparse_top_5_categorical_accuracy: 0.0100 - val_sparse_top_10_categorical_accuracy: 0.0100
traincallback: INFO     epoch: 0 - val_loss: 1892.6826123046876 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 7.371352685292562 val_mean_anwer_guess: 473.1333333333333 val_sparse_top_2_categorical_accuracy: 0.006666666666666667 val_sparse_top_5_categorical_accuracy: 0.01 val_sparse_top_10_categorical_accuracy: 0.01 loss: 72279.29423828125 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 7.826378536224365 mean_anwer_guess: 451.15 sparse_top_2_categorical_accuracy: 0.003125 sparse_top_5_categorical_accuracy: 0.00703125 sparse_top_10_categorical_accuracy: 0.0109375 lr: 0.0005 
Epoch 2/150
20/20 [==============================] - 331s 17s/step - loss: 22305.4429 - sparse_categorical_accuracy: 0.0031 - sparse_categorical_crossentropy: 7.3037 - mean_anwer_guess: 519.3000 - sparse_top_2_categorical_accuracy: 0.0047 - sparse_top_5_categorical_accuracy: 0.0055 - sparse_top_10_categorical_accuracy: 0.0133 - val_loss: 1158.1500 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 7.1955 - val_mean_anwer_guess: 496.5200 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0033
traincallback: INFO     epoch: 1 - val_loss: 1158.1499674479167 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 7.195486653645833 val_mean_anwer_guess: 496.52 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.0033333333333333335 loss: 22305.44287109375 sparse_categorical_accuracy: 0.003125 sparse_categorical_crossentropy: 7.303728413581848 mean_anwer_guess: 519.3 sparse_top_2_categorical_accuracy: 0.0046875 sparse_top_5_categorical_accuracy: 0.00546875 sparse_top_10_categorical_accuracy: 0.01328125 lr: 0.0005 
Epoch 3/150
20/20 [==============================] - 327s 16s/step - loss: 14746.6170 - sparse_categorical_accuracy: 0.0023 - sparse_categorical_crossentropy: 6.9962 - mean_anwer_guess: 366.9000 - sparse_top_2_categorical_accuracy: 0.0023 - sparse_top_5_categorical_accuracy: 0.0055 - sparse_top_10_categorical_accuracy: 0.0117 - val_loss: 713.7605 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 7.0041 - val_mean_anwer_guess: 535.3333 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0000e+00 - val_sparse_top_10_categorical_accuracy: 0.0067
traincallback: INFO     epoch: 2 - val_loss: 713.7604996744792 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 7.004112447102864 val_mean_anwer_guess: 535.3333333333334 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0 val_sparse_top_10_categorical_accuracy: 0.006666666666666667 loss: 14746.6169921875 sparse_categorical_accuracy: 0.00234375 sparse_categorical_crossentropy: 6.996247887611389 mean_anwer_guess: 366.9 sparse_top_2_categorical_accuracy: 0.00234375 sparse_top_5_categorical_accuracy: 0.00546875 sparse_top_10_categorical_accuracy: 0.01171875 lr: 0.0005 
Epoch 4/150
20/20 [==============================] - 332s 17s/step - loss: 11757.3366 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 6.8674 - mean_anwer_guess: 461.7000 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0117 - sparse_top_10_categorical_accuracy: 0.0227 - val_loss: 477.2980 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9355 - val_mean_anwer_guess: 510.4133 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0133
traincallback: INFO     epoch: 3 - val_loss: 477.29796549479164 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.935500621795654 val_mean_anwer_guess: 510.41333333333336 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.013333333333333334 loss: 11757.33662109375 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 6.867412948608399 mean_anwer_guess: 461.7 sparse_top_2_categorical_accuracy: 0.00390625 sparse_top_5_categorical_accuracy: 0.01171875 sparse_top_10_categorical_accuracy: 0.02265625 lr: 0.0005 
Epoch 5/150
20/20 [==============================] - 313s 16s/step - loss: 9759.2302 - sparse_categorical_accuracy: 0.0031 - sparse_categorical_crossentropy: 6.8926 - mean_anwer_guess: 527.5500 - sparse_top_2_categorical_accuracy: 0.0047 - sparse_top_5_categorical_accuracy: 0.0117 - sparse_top_10_categorical_accuracy: 0.0219 - val_loss: 349.2577 - val_sparse_categorical_accuracy: 0.0033 - val_sparse_categorical_crossentropy: 6.9212 - val_mean_anwer_guess: 458.2133 - val_sparse_top_2_categorical_accuracy: 0.0033 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0167
traincallback: INFO     epoch: 4 - val_loss: 349.25768595377605 val_sparse_categorical_accuracy: 0.0033333333333333335 val_sparse_categorical_crossentropy: 6.921210231781006 val_mean_anwer_guess: 458.2133333333333 val_sparse_top_2_categorical_accuracy: 0.0033333333333333335 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.016666666666666666 loss: 9759.230224609375 sparse_categorical_accuracy: 0.003125 sparse_categorical_crossentropy: 6.892594838142395 mean_anwer_guess: 527.55 sparse_top_2_categorical_accuracy: 0.0046875 sparse_top_5_categorical_accuracy: 0.01171875 sparse_top_10_categorical_accuracy: 0.021875 lr: 0.0005 
Epoch 6/150
20/20 [==============================] - 314s 16s/step - loss: 8383.8431 - sparse_categorical_accuracy: 0.0039 - sparse_categorical_crossentropy: 6.9110 - mean_anwer_guess: 237.0000 - sparse_top_2_categorical_accuracy: 0.0055 - sparse_top_5_categorical_accuracy: 0.0141 - sparse_top_10_categorical_accuracy: 0.0266 - val_loss: 276.3368 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9180 - val_mean_anwer_guess: 493.2267 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0100
traincallback: INFO     epoch: 5 - val_loss: 276.3368009440104 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.917951691945394 val_mean_anwer_guess: 493.2266666666667 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.01 loss: 8383.843115234375 sparse_categorical_accuracy: 0.00390625 sparse_categorical_crossentropy: 6.910976839065552 mean_anwer_guess: 237.0 sparse_top_2_categorical_accuracy: 0.00546875 sparse_top_5_categorical_accuracy: 0.0140625 sparse_top_10_categorical_accuracy: 0.0265625 lr: 0.0005 
Epoch 7/150
20/20 [==============================] - 316s 16s/step - loss: 7551.0763 - sparse_categorical_accuracy: 0.0031 - sparse_categorical_crossentropy: 6.9137 - mean_anwer_guess: 513.4500 - sparse_top_2_categorical_accuracy: 0.0047 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0203 - val_loss: 233.6073 - val_sparse_categorical_accuracy: 0.0033 - val_sparse_categorical_crossentropy: 6.9181 - val_mean_anwer_guess: 481.6667 - val_sparse_top_2_categorical_accuracy: 0.0033 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0133
traincallback: INFO     epoch: 6 - val_loss: 233.60728352864584 val_sparse_categorical_accuracy: 0.0033333333333333335 val_sparse_categorical_crossentropy: 6.918062299092611 val_mean_anwer_guess: 481.6666666666667 val_sparse_top_2_categorical_accuracy: 0.0033333333333333335 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.013333333333333334 loss: 7551.076342773437 sparse_categorical_accuracy: 0.003125 sparse_categorical_crossentropy: 6.913731050491333 mean_anwer_guess: 513.45 sparse_top_2_categorical_accuracy: 0.0046875 sparse_top_5_categorical_accuracy: 0.00859375 sparse_top_10_categorical_accuracy: 0.0203125 lr: 0.0005 
Epoch 8/150
20/20 [==============================] - 320s 16s/step - loss: 7037.9725 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 6.9135 - mean_anwer_guess: 433.8500 - sparse_top_2_categorical_accuracy: 0.0016 - sparse_top_5_categorical_accuracy: 0.0055 - sparse_top_10_categorical_accuracy: 0.0133 - val_loss: 250.0103 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9175 - val_mean_anwer_guess: 504.5200 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0000e+00 - val_sparse_top_10_categorical_accuracy: 0.0067
traincallback: INFO     epoch: 7 - val_loss: 250.01030049641926 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.917485218048096 val_mean_anwer_guess: 504.52 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0 val_sparse_top_10_categorical_accuracy: 0.006666666666666667 loss: 7037.9724609375 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 6.913473224639892 mean_anwer_guess: 433.85 sparse_top_2_categorical_accuracy: 0.0015625 sparse_top_5_categorical_accuracy: 0.00546875 sparse_top_10_categorical_accuracy: 0.01328125 lr: 0.0005 
Epoch 9/150
20/20 [==============================] - 321s 16s/step - loss: 6745.8842 - sparse_categorical_accuracy: 0.0000e+00 - sparse_categorical_crossentropy: 6.9122 - mean_anwer_guess: 461.6500 - sparse_top_2_categorical_accuracy: 0.0031 - sparse_top_5_categorical_accuracy: 0.0156 - sparse_top_10_categorical_accuracy: 0.0266 - val_loss: 207.7814 - val_sparse_categorical_accuracy: 0.0033 - val_sparse_categorical_crossentropy: 6.9172 - val_mean_anwer_guess: 500.9467 - val_sparse_top_2_categorical_accuracy: 0.0033 - val_sparse_top_5_categorical_accuracy: 0.0067 - val_sparse_top_10_categorical_accuracy: 0.0167
traincallback: INFO     epoch: 8 - val_loss: 207.78143046061197 val_sparse_categorical_accuracy: 0.0033333333333333335 val_sparse_categorical_crossentropy: 6.917235730489095 val_mean_anwer_guess: 500.94666666666666 val_sparse_top_2_categorical_accuracy: 0.0033333333333333335 val_sparse_top_5_categorical_accuracy: 0.006666666666666667 val_sparse_top_10_categorical_accuracy: 0.016666666666666666 loss: 6745.8841796875 sparse_categorical_accuracy: 0.0 sparse_categorical_crossentropy: 6.912249827384949 mean_anwer_guess: 461.65 sparse_top_2_categorical_accuracy: 0.003125 sparse_top_5_categorical_accuracy: 0.015625 sparse_top_10_categorical_accuracy: 0.0265625 lr: 0.0005 
Epoch 10/150
20/20 [==============================] - 322s 16s/step - loss: 6187.9865 - sparse_categorical_accuracy: 0.0031 - sparse_categorical_crossentropy: 6.9120 - mean_anwer_guess: 459.1500 - sparse_top_2_categorical_accuracy: 0.0047 - sparse_top_5_categorical_accuracy: 0.0102 - sparse_top_10_categorical_accuracy: 0.0195 - val_loss: 201.7798 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9166 - val_mean_anwer_guess: 451.2800 - val_sparse_top_2_categorical_accuracy: 0.0033 - val_sparse_top_5_categorical_accuracy: 0.0067 - val_sparse_top_10_categorical_accuracy: 0.0067
traincallback: INFO     epoch: 9 - val_loss: 201.77976094563803 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.916606928507487 val_mean_anwer_guess: 451.28 val_sparse_top_2_categorical_accuracy: 0.0033333333333333335 val_sparse_top_5_categorical_accuracy: 0.006666666666666667 val_sparse_top_10_categorical_accuracy: 0.006666666666666667 loss: 6187.986499023437 sparse_categorical_accuracy: 0.003125 sparse_categorical_crossentropy: 6.911971855163574 mean_anwer_guess: 459.15 sparse_top_2_categorical_accuracy: 0.0046875 sparse_top_5_categorical_accuracy: 0.01015625 sparse_top_10_categorical_accuracy: 0.01953125 lr: 0.0005 
Epoch 11/150
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3030] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3030] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3031] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3031] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3032] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3032] is not in the [index]'
20/20 [==============================] - 312s 16s/step - loss: 6009.7854 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.9112 - mean_anwer_guess: 367.0500 - sparse_top_2_categorical_accuracy: 0.0031 - sparse_top_5_categorical_accuracy: 0.0094 - sparse_top_10_categorical_accuracy: 0.0187 - val_loss: 179.8454 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9179 - val_mean_anwer_guess: 437.6667 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0067
traincallback: INFO     epoch: 10 - val_loss: 179.8454327392578 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.917929712931315 val_mean_anwer_guess: 437.6666666666667 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.006666666666666667 loss: 6009.785375976562 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.9111556053161625 mean_anwer_guess: 367.05 sparse_top_2_categorical_accuracy: 0.003125 sparse_top_5_categorical_accuracy: 0.009375 sparse_top_10_categorical_accuracy: 0.01875 lr: 0.0005 
Epoch 12/150
20/20 [==============================] - 321s 16s/step - loss: 5702.8190 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.9107 - mean_anwer_guess: 449.5000 - sparse_top_2_categorical_accuracy: 0.0016 - sparse_top_5_categorical_accuracy: 0.0055 - sparse_top_10_categorical_accuracy: 0.0172 - val_loss: 180.1914 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9188 - val_mean_anwer_guess: 521.4000 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0000e+00 - val_sparse_top_10_categorical_accuracy: 0.0100
traincallback: INFO     epoch: 11 - val_loss: 180.19140665690105 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.918752930959066 val_mean_anwer_guess: 521.4 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0 val_sparse_top_10_categorical_accuracy: 0.01 loss: 5702.8189697265625 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.910686731338501 mean_anwer_guess: 449.5 sparse_top_2_categorical_accuracy: 0.0015625 sparse_top_5_categorical_accuracy: 0.00546875 sparse_top_10_categorical_accuracy: 0.0171875 lr: 0.0005 
Epoch 13/150
20/20 [==============================] - 334s 17s/step - loss: 5418.8233 - sparse_categorical_accuracy: 0.0000e+00 - sparse_categorical_crossentropy: 6.9098 - mean_anwer_guess: 411.1500 - sparse_top_2_categorical_accuracy: 7.8125e-04 - sparse_top_5_categorical_accuracy: 0.0109 - sparse_top_10_categorical_accuracy: 0.0211 - val_loss: 168.6256 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9169 - val_mean_anwer_guess: 554.9200 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0067 - val_sparse_top_10_categorical_accuracy: 0.0067
traincallback: INFO     epoch: 12 - val_loss: 168.62560038248697 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.916946201324463 val_mean_anwer_guess: 554.92 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.006666666666666667 val_sparse_top_10_categorical_accuracy: 0.006666666666666667 loss: 5418.823291015625 sparse_categorical_accuracy: 0.0 sparse_categorical_crossentropy: 6.909760808944702 mean_anwer_guess: 411.15 sparse_top_2_categorical_accuracy: 0.00078125 sparse_top_5_categorical_accuracy: 0.0109375 sparse_top_10_categorical_accuracy: 0.02109375 lr: 0.0005 
Epoch 14/150
20/20 [==============================] - 331s 17s/step - loss: 5436.2248 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.9085 - mean_anwer_guess: 517.9500 - sparse_top_2_categorical_accuracy: 0.0023 - sparse_top_5_categorical_accuracy: 0.0094 - sparse_top_10_categorical_accuracy: 0.0148 - val_loss: 186.9243 - val_sparse_categorical_accuracy: 0.0067 - val_sparse_categorical_crossentropy: 6.9170 - val_mean_anwer_guess: 492.3333 - val_sparse_top_2_categorical_accuracy: 0.0067 - val_sparse_top_5_categorical_accuracy: 0.0067 - val_sparse_top_10_categorical_accuracy: 0.0100
traincallback: INFO     epoch: 13 - val_loss: 186.92434041341147 val_sparse_categorical_accuracy: 0.006666666666666667 val_sparse_categorical_crossentropy: 6.916962973276774 val_mean_anwer_guess: 492.3333333333333 val_sparse_top_2_categorical_accuracy: 0.006666666666666667 val_sparse_top_5_categorical_accuracy: 0.006666666666666667 val_sparse_top_10_categorical_accuracy: 0.01 loss: 5436.224755859375 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.908514475822448 mean_anwer_guess: 517.95 sparse_top_2_categorical_accuracy: 0.00234375 sparse_top_5_categorical_accuracy: 0.009375 sparse_top_10_categorical_accuracy: 0.01484375 lr: 0.0005 
Epoch 15/150
20/20 [==============================] - 323s 16s/step - loss: 5270.2783 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 6.9088 - mean_anwer_guess: 505.9500 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0187 - val_loss: 180.5704 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9181 - val_mean_anwer_guess: 524.0800 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0000e+00 - val_sparse_top_10_categorical_accuracy: 0.0033
traincallback: INFO     epoch: 14 - val_loss: 180.57043375651043 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.918078174591065 val_mean_anwer_guess: 524.08 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0 val_sparse_top_10_categorical_accuracy: 0.0033333333333333335 loss: 5270.278271484375 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 6.90878643989563 mean_anwer_guess: 505.95 sparse_top_2_categorical_accuracy: 0.00390625 sparse_top_5_categorical_accuracy: 0.00859375 sparse_top_10_categorical_accuracy: 0.01875 lr: 0.0005 
Epoch 16/150
20/20 [==============================] - 325s 16s/step - loss: 5393.2199 - sparse_categorical_accuracy: 0.0023 - sparse_categorical_crossentropy: 6.9075 - mean_anwer_guess: 470.8000 - sparse_top_2_categorical_accuracy: 0.0023 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0180 - val_loss: 179.1251 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9160 - val_mean_anwer_guess: 521.4133 - val_sparse_top_2_categorical_accuracy: 0.0033 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0067
traincallback: INFO     epoch: 15 - val_loss: 179.12506388346355 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.9159996096293135 val_mean_anwer_guess: 521.4133333333333 val_sparse_top_2_categorical_accuracy: 0.0033333333333333335 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.006666666666666667 loss: 5393.219946289062 sparse_categorical_accuracy: 0.00234375 sparse_categorical_crossentropy: 6.90752956867218 mean_anwer_guess: 470.8 sparse_top_2_categorical_accuracy: 0.00234375 sparse_top_5_categorical_accuracy: 0.00859375 sparse_top_10_categorical_accuracy: 0.01796875 lr: 0.0005 
Epoch 17/150
20/20 [==============================] - 318s 16s/step - loss: 4978.4317 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.9074 - mean_anwer_guess: 443.1500 - sparse_top_2_categorical_accuracy: 0.0031 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0203 - val_loss: 221.3766 - val_sparse_categorical_accuracy: 0.0033 - val_sparse_categorical_crossentropy: 6.9185 - val_mean_anwer_guess: 545.5333 - val_sparse_top_2_categorical_accuracy: 0.0067 - val_sparse_top_5_categorical_accuracy: 0.0067 - val_sparse_top_10_categorical_accuracy: 0.0133
traincallback: INFO     epoch: 16 - val_loss: 221.37662109375 val_sparse_categorical_accuracy: 0.0033333333333333335 val_sparse_categorical_crossentropy: 6.918507448832194 val_mean_anwer_guess: 545.5333333333333 val_sparse_top_2_categorical_accuracy: 0.006666666666666667 val_sparse_top_5_categorical_accuracy: 0.006666666666666667 val_sparse_top_10_categorical_accuracy: 0.013333333333333334 loss: 4978.431665039063 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.9073686599731445 mean_anwer_guess: 443.15 sparse_top_2_categorical_accuracy: 0.003125 sparse_top_5_categorical_accuracy: 0.00859375 sparse_top_10_categorical_accuracy: 0.0203125 lr: 0.0005 
Epoch 18/150
20/20 [==============================] - 327s 16s/step - loss: 5361.6376 - sparse_categorical_accuracy: 0.0023 - sparse_categorical_crossentropy: 6.9069 - mean_anwer_guess: 588.4000 - sparse_top_2_categorical_accuracy: 0.0031 - sparse_top_5_categorical_accuracy: 0.0094 - sparse_top_10_categorical_accuracy: 0.0172 - val_loss: 164.1036 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9183 - val_mean_anwer_guess: 473.7467 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0133
traincallback: INFO     epoch: 17 - val_loss: 164.10355163574218 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.918310362497966 val_mean_anwer_guess: 473.74666666666667 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.013333333333333334 loss: 5361.637622070312 sparse_categorical_accuracy: 0.00234375 sparse_categorical_crossentropy: 6.906924247741699 mean_anwer_guess: 588.4 sparse_top_2_categorical_accuracy: 0.003125 sparse_top_5_categorical_accuracy: 0.009375 sparse_top_10_categorical_accuracy: 0.0171875 lr: 0.0005 
Epoch 19/150
20/20 [==============================] - 323s 16s/step - loss: 4651.0018 - sparse_categorical_accuracy: 0.0047 - sparse_categorical_crossentropy: 6.9048 - mean_anwer_guess: 447.4500 - sparse_top_2_categorical_accuracy: 0.0055 - sparse_top_5_categorical_accuracy: 0.0094 - sparse_top_10_categorical_accuracy: 0.0203 - val_loss: 146.1939 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9199 - val_mean_anwer_guess: 490.9600 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0100
traincallback: INFO     epoch: 18 - val_loss: 146.19388153076173 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.919873574574789 val_mean_anwer_guess: 490.96 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.01 loss: 4651.0018310546875 sparse_categorical_accuracy: 0.0046875 sparse_categorical_crossentropy: 6.904846167564392 mean_anwer_guess: 447.45 sparse_top_2_categorical_accuracy: 0.00546875 sparse_top_5_categorical_accuracy: 0.009375 sparse_top_10_categorical_accuracy: 0.0203125 lr: 0.0005 
Epoch 20/150
20/20 [==============================] - 325s 16s/step - loss: 5107.9682 - sparse_categorical_accuracy: 0.0023 - sparse_categorical_crossentropy: 6.9058 - mean_anwer_guess: 463.1500 - sparse_top_2_categorical_accuracy: 0.0031 - sparse_top_5_categorical_accuracy: 0.0047 - sparse_top_10_categorical_accuracy: 0.0133 - val_loss: 153.7441 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9174 - val_mean_anwer_guess: 514.1467 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0067

Epoch 00020: val_sparse_categorical_accuracy improved from -inf to 0.00000, saving model to inaturalist_xception.h5
traincallback: INFO     epoch: 19 - val_loss: 153.74413940429687 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.917436389923096 val_mean_anwer_guess: 514.1466666666666 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.006666666666666667 loss: 5107.968176269531 sparse_categorical_accuracy: 0.00234375 sparse_categorical_crossentropy: 6.905842852592468 mean_anwer_guess: 463.15 sparse_top_2_categorical_accuracy: 0.003125 sparse_top_5_categorical_accuracy: 0.0046875 sparse_top_10_categorical_accuracy: 0.01328125 lr: 0.0005 
Epoch 21/150
19/20 [===========================>..] - ETA: 12s - loss: 4671.2478 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.9031 - mean_anwer_guess: 528.1053 - sparse_top_2_categorical_accuracy: 0.0041 - sparse_top_5_categorical_accuracy: 0.0082 - sparse_top_10_categorical_accuracy: 0.0132
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3030] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3030] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3031] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3031] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3032] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3032] is not in the [index]'
20/20 [==============================] - 323s 16s/step - loss: 4658.8865 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.9031 - mean_anwer_guess: 510.5000 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0078 - sparse_top_10_categorical_accuracy: 0.0125 - val_loss: 140.8388 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9200 - val_mean_anwer_guess: 471.5333 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0000e+00 - val_sparse_top_10_categorical_accuracy: 0.0000e+00
traincallback: INFO     epoch: 20 - val_loss: 140.83882486979167 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.919992713928223 val_mean_anwer_guess: 471.53333333333336 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0 val_sparse_top_10_categorical_accuracy: 0.0 loss: 4658.886547851563 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.903148984909057 mean_anwer_guess: 510.5 sparse_top_2_categorical_accuracy: 0.00390625 sparse_top_5_categorical_accuracy: 0.0078125 sparse_top_10_categorical_accuracy: 0.0125 lr: 0.0005 
Epoch 22/150
20/20 [==============================] - 333s 17s/step - loss: 4933.3177 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.9026 - mean_anwer_guess: 446.8000 - sparse_top_2_categorical_accuracy: 0.0031 - sparse_top_5_categorical_accuracy: 0.0055 - sparse_top_10_categorical_accuracy: 0.0141 - val_loss: 170.8401 - val_sparse_categorical_accuracy: 0.0033 - val_sparse_categorical_crossentropy: 6.9207 - val_mean_anwer_guess: 514.9200 - val_sparse_top_2_categorical_accuracy: 0.0067 - val_sparse_top_5_categorical_accuracy: 0.0133 - val_sparse_top_10_categorical_accuracy: 0.0200
traincallback: INFO     epoch: 21 - val_loss: 170.8401141357422 val_sparse_categorical_accuracy: 0.0033333333333333335 val_sparse_categorical_crossentropy: 6.920669473012288 val_mean_anwer_guess: 514.92 val_sparse_top_2_categorical_accuracy: 0.006666666666666667 val_sparse_top_5_categorical_accuracy: 0.013333333333333334 val_sparse_top_10_categorical_accuracy: 0.02 loss: 4933.317687988281 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.9026219844818115 mean_anwer_guess: 446.8 sparse_top_2_categorical_accuracy: 0.003125 sparse_top_5_categorical_accuracy: 0.00546875 sparse_top_10_categorical_accuracy: 0.0140625 lr: 0.0005 
Epoch 23/150
20/20 [==============================] - 338s 17s/step - loss: 4780.6695 - sparse_categorical_accuracy: 0.0031 - sparse_categorical_crossentropy: 6.9022 - mean_anwer_guess: 466.6000 - sparse_top_2_categorical_accuracy: 0.0047 - sparse_top_5_categorical_accuracy: 0.0094 - sparse_top_10_categorical_accuracy: 0.0180 - val_loss: 190.9515 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9179 - val_mean_anwer_guess: 502.5733 - val_sparse_top_2_categorical_accuracy: 0.0033 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0133
traincallback: INFO     epoch: 22 - val_loss: 190.95153991699217 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.9179022471110025 val_mean_anwer_guess: 502.5733333333333 val_sparse_top_2_categorical_accuracy: 0.0033333333333333335 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.013333333333333334 loss: 4780.669470214843 sparse_categorical_accuracy: 0.003125 sparse_categorical_crossentropy: 6.902244973182678 mean_anwer_guess: 466.6 sparse_top_2_categorical_accuracy: 0.0046875 sparse_top_5_categorical_accuracy: 0.009375 sparse_top_10_categorical_accuracy: 0.01796875 lr: 0.0005 
Epoch 24/150
20/20 [==============================] - 331s 17s/step - loss: 4796.8529 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.9015 - mean_anwer_guess: 393.7000 - sparse_top_2_categorical_accuracy: 0.0016 - sparse_top_5_categorical_accuracy: 0.0102 - sparse_top_10_categorical_accuracy: 0.0180 - val_loss: 179.2027 - val_sparse_categorical_accuracy: 0.0033 - val_sparse_categorical_crossentropy: 6.9161 - val_mean_anwer_guess: 473.8400 - val_sparse_top_2_categorical_accuracy: 0.0033 - val_sparse_top_5_categorical_accuracy: 0.0067 - val_sparse_top_10_categorical_accuracy: 0.0133
traincallback: INFO     epoch: 23 - val_loss: 179.20270853678386 val_sparse_categorical_accuracy: 0.0033333333333333335 val_sparse_categorical_crossentropy: 6.916128889719645 val_mean_anwer_guess: 473.84 val_sparse_top_2_categorical_accuracy: 0.0033333333333333335 val_sparse_top_5_categorical_accuracy: 0.006666666666666667 val_sparse_top_10_categorical_accuracy: 0.013333333333333334 loss: 4796.852868652344 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.9015449523925785 mean_anwer_guess: 393.7 sparse_top_2_categorical_accuracy: 0.0015625 sparse_top_5_categorical_accuracy: 0.01015625 sparse_top_10_categorical_accuracy: 0.01796875 lr: 0.0005 
Epoch 00024: ReduceLROnPlateau reducing learning rate to 0.0003000000142492354.
Epoch 25/150
20/20 [==============================] - 327s 16s/step - loss: 3995.6191 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 6.9012 - mean_anwer_guess: 497.9000 - sparse_top_2_categorical_accuracy: 0.0016 - sparse_top_5_categorical_accuracy: 0.0078 - sparse_top_10_categorical_accuracy: 0.0133 - val_loss: 119.3189 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9167 - val_mean_anwer_guess: 502.7200 - val_sparse_top_2_categorical_accuracy: 0.0033 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0067
traincallback: INFO     epoch: 24 - val_loss: 119.31890045166016 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.916702817281087 val_mean_anwer_guess: 502.72 val_sparse_top_2_categorical_accuracy: 0.0033333333333333335 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.006666666666666667 loss: 3995.6191162109376 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 6.901170468330383 mean_anwer_guess: 497.9 sparse_top_2_categorical_accuracy: 0.0015625 sparse_top_5_categorical_accuracy: 0.0078125 sparse_top_10_categorical_accuracy: 0.01328125 lr: 0.0003 
Epoch 26/150
20/20 [==============================] - 328s 16s/step - loss: 3407.6282 - sparse_categorical_accuracy: 0.0000e+00 - sparse_categorical_crossentropy: 6.9009 - mean_anwer_guess: 325.7000 - sparse_top_2_categorical_accuracy: 0.0000e+00 - sparse_top_5_categorical_accuracy: 0.0070 - sparse_top_10_categorical_accuracy: 0.0180 - val_loss: 107.8284 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9181 - val_mean_anwer_guess: 417.0800 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0100 - val_sparse_top_10_categorical_accuracy: 0.0133
traincallback: INFO     epoch: 25 - val_loss: 107.82840647379557 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.9180589866638185 val_mean_anwer_guess: 417.08 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.01 val_sparse_top_10_categorical_accuracy: 0.013333333333333334 loss: 3407.6282470703127 sparse_categorical_accuracy: 0.0 sparse_categorical_crossentropy: 6.900906300544738 mean_anwer_guess: 325.7 sparse_top_2_categorical_accuracy: 0.0 sparse_top_5_categorical_accuracy: 0.00703125 sparse_top_10_categorical_accuracy: 0.01796875 lr: 0.0003 
Epoch 27/150
20/20 [==============================] - 330s 16s/step - loss: 3168.7803 - sparse_categorical_accuracy: 0.0031 - sparse_categorical_crossentropy: 6.8998 - mean_anwer_guess: 420.0000 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0078 - sparse_top_10_categorical_accuracy: 0.0172 - val_loss: 98.6710 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9190 - val_mean_anwer_guess: 526.6800 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0067
traincallback: INFO     epoch: 26 - val_loss: 98.67104654947917 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.918981240590413 val_mean_anwer_guess: 526.68 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.006666666666666667 loss: 3168.7803466796877 sparse_categorical_accuracy: 0.003125 sparse_categorical_crossentropy: 6.899770498275757 mean_anwer_guess: 420.0 sparse_top_2_categorical_accuracy: 0.00390625 sparse_top_5_categorical_accuracy: 0.0078125 sparse_top_10_categorical_accuracy: 0.0171875 lr: 0.0003 
Epoch 28/150
20/20 [==============================] - 342s 17s/step - loss: 3400.8443 - sparse_categorical_accuracy: 0.0031 - sparse_categorical_crossentropy: 6.8982 - mean_anwer_guess: 461.4000 - sparse_top_2_categorical_accuracy: 0.0055 - sparse_top_5_categorical_accuracy: 0.0125 - sparse_top_10_categorical_accuracy: 0.0258 - val_loss: 94.9354 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9178 - val_mean_anwer_guess: 458.9067 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0000e+00 - val_sparse_top_10_categorical_accuracy: 0.0033
traincallback: INFO     epoch: 27 - val_loss: 94.93542175292968 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.917813180287679 val_mean_anwer_guess: 458.9066666666667 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0 val_sparse_top_10_categorical_accuracy: 0.0033333333333333335 loss: 3400.8443359375 sparse_categorical_accuracy: 0.003125 sparse_categorical_crossentropy: 6.898177647590638 mean_anwer_guess: 461.4 sparse_top_2_categorical_accuracy: 0.00546875 sparse_top_5_categorical_accuracy: 0.0125 sparse_top_10_categorical_accuracy: 0.02578125 lr: 0.0003 
Epoch 29/150
20/20 [==============================] - 322s 16s/step - loss: 3169.0374 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.8987 - mean_anwer_guess: 577.0000 - sparse_top_2_categorical_accuracy: 0.0023 - sparse_top_5_categorical_accuracy: 0.0109 - sparse_top_10_categorical_accuracy: 0.0164 - val_loss: 95.6620 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9161 - val_mean_anwer_guess: 457.8800 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0033
traincallback: INFO     epoch: 28 - val_loss: 95.66195810953776 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.916140003204346 val_mean_anwer_guess: 457.88 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.0033333333333333335 loss: 3169.0373779296874 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.898715519905091 mean_anwer_guess: 577.0 sparse_top_2_categorical_accuracy: 0.00234375 sparse_top_5_categorical_accuracy: 0.0109375 sparse_top_10_categorical_accuracy: 0.01640625 lr: 0.0003 
Epoch 30/150
20/20 [==============================] - 329s 16s/step - loss: 3284.2957 - sparse_categorical_accuracy: 0.0031 - sparse_categorical_crossentropy: 6.8982 - mean_anwer_guess: 441.1500 - sparse_top_2_categorical_accuracy: 0.0063 - sparse_top_5_categorical_accuracy: 0.0141 - sparse_top_10_categorical_accuracy: 0.0242 - val_loss: 95.3214 - val_sparse_categorical_accuracy: 0.0033 - val_sparse_categorical_crossentropy: 6.9179 - val_mean_anwer_guess: 498.9200 - val_sparse_top_2_categorical_accuracy: 0.0033 - val_sparse_top_5_categorical_accuracy: 0.0067 - val_sparse_top_10_categorical_accuracy: 0.0100
traincallback: INFO     epoch: 29 - val_loss: 95.32142049153646 val_sparse_categorical_accuracy: 0.0033333333333333335 val_sparse_categorical_crossentropy: 6.917855644226075 val_mean_anwer_guess: 498.92 val_sparse_top_2_categorical_accuracy: 0.0033333333333333335 val_sparse_top_5_categorical_accuracy: 0.006666666666666667 val_sparse_top_10_categorical_accuracy: 0.01 loss: 3284.295703125 sparse_categorical_accuracy: 0.003125 sparse_categorical_crossentropy: 6.8982326030731205 mean_anwer_guess: 441.15 sparse_top_2_categorical_accuracy: 0.00625 sparse_top_5_categorical_accuracy: 0.0140625 sparse_top_10_categorical_accuracy: 0.02421875 lr: 0.0003 
Epoch 31/150
19/20 [===========================>..] - ETA: 12s - loss: 3104.8761 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.8976 - mean_anwer_guess: 481.2632 - sparse_top_2_categorical_accuracy: 0.0049 - sparse_top_5_categorical_accuracy: 0.0090 - sparse_top_10_categorical_accuracy: 0.0189
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3030] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3030] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3031] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3031] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3032] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3032] is not in the [index]'
20/20 [==============================] - 316s 16s/step - loss: 3102.6513 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.8976 - mean_anwer_guess: 505.1500 - sparse_top_2_categorical_accuracy: 0.0047 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0187 - val_loss: 113.0610 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9211 - val_mean_anwer_guess: 425.9467 - val_sparse_top_2_categorical_accuracy: 0.0033 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0067
traincallback: INFO     epoch: 30 - val_loss: 113.06103647867839 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.921059652964274 val_mean_anwer_guess: 425.94666666666666 val_sparse_top_2_categorical_accuracy: 0.0033333333333333335 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.006666666666666667 loss: 3102.65126953125 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.897599792480468 mean_anwer_guess: 505.15 sparse_top_2_categorical_accuracy: 0.0046875 sparse_top_5_categorical_accuracy: 0.00859375 sparse_top_10_categorical_accuracy: 0.01875 lr: 0.0003 
Epoch 32/150
20/20 [==============================] - 322s 16s/step - loss: 3015.0873 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.8968 - mean_anwer_guess: 426.7500 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0164 - val_loss: 91.9944 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9179 - val_mean_anwer_guess: 525.2800 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0000e+00 - val_sparse_top_10_categorical_accuracy: 0.0033
traincallback: INFO     epoch: 31 - val_loss: 91.99441772460938 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.91790880203247 val_mean_anwer_guess: 525.28 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0 val_sparse_top_10_categorical_accuracy: 0.0033333333333333335 loss: 3015.0873168945313 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.896802258491516 mean_anwer_guess: 426.75 sparse_top_2_categorical_accuracy: 0.00390625 sparse_top_5_categorical_accuracy: 0.00859375 sparse_top_10_categorical_accuracy: 0.01640625 lr: 0.0003 
Epoch 33/150
20/20 [==============================] - 327s 16s/step - loss: 3070.9360 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 6.8980 - mean_anwer_guess: 474.6000 - sparse_top_2_categorical_accuracy: 0.0023 - sparse_top_5_categorical_accuracy: 0.0055 - sparse_top_10_categorical_accuracy: 0.0141 - val_loss: 95.6509 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9154 - val_mean_anwer_guess: 470.7600 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0000e+00 - val_sparse_top_10_categorical_accuracy: 0.0133
traincallback: INFO     epoch: 32 - val_loss: 95.65090382893881 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.9153985595703125 val_mean_anwer_guess: 470.76 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0 val_sparse_top_10_categorical_accuracy: 0.013333333333333334 loss: 3070.9359619140623 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 6.897995805740356 mean_anwer_guess: 474.6 sparse_top_2_categorical_accuracy: 0.00234375 sparse_top_5_categorical_accuracy: 0.00546875 sparse_top_10_categorical_accuracy: 0.0140625 lr: 0.0003 
Epoch 34/150
20/20 [==============================] - 326s 16s/step - loss: 3158.7775 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 6.8959 - mean_anwer_guess: 486.2000 - sparse_top_2_categorical_accuracy: 0.0031 - sparse_top_5_categorical_accuracy: 0.0063 - sparse_top_10_categorical_accuracy: 0.0180 - val_loss: 95.4145 - val_sparse_categorical_accuracy: 0.0033 - val_sparse_categorical_crossentropy: 6.9177 - val_mean_anwer_guess: 497.6400 - val_sparse_top_2_categorical_accuracy: 0.0033 - val_sparse_top_5_categorical_accuracy: 0.0100 - val_sparse_top_10_categorical_accuracy: 0.0133
Restoring model weights from the end of the best epoch
traincallback: INFO     epoch: 33 - val_loss: 95.41447835286458 val_sparse_categorical_accuracy: 0.0033333333333333335 val_sparse_categorical_crossentropy: 6.917749404907227 val_mean_anwer_guess: 497.64 val_sparse_top_2_categorical_accuracy: 0.0033333333333333335 val_sparse_top_5_categorical_accuracy: 0.01 val_sparse_top_10_categorical_accuracy: 0.013333333333333334 loss: 3158.7774658203125 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 6.89590904712677 mean_anwer_guess: 486.2 sparse_top_2_categorical_accuracy: 0.003125 sparse_top_5_categorical_accuracy: 0.00625 sparse_top_10_categorical_accuracy: 0.01796875 lr: 0.0003 
Restoring model weights from the end of the best epoch
traincallback: INFO     training end: model_1
Epoch 00034: early stopping
Epoch 00034: early stopping
root        : INFO     pre train end 3:05:34.855935
root        : INFO     remaining time 5:54:25.142503
In [26]:
pretrained_model.trainable=True
optimizer=keras.optimizers.Adam(clipnorm=5. , clipvalue=5.,amsgrad=True)
model.compile(optimizer,
                            loss="sparse_categorical_crossentropy",
              metrics=model.metrics
             )
model.summary()
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
image_input (InputLayer)        (None, None, None, 3 0                                            
__________________________________________________________________________________________________
xception (Model)                (None, None, None, 2 20861480    image_input[0][0]                
__________________________________________________________________________________________________
conv2d_5 (Conv2D)               (None, None, None, 6 11059800    xception[1][0]                   
__________________________________________________________________________________________________
global_max_pooling2d_2 (GlobalM (None, 600)          0           conv2d_5[0][0]                   
__________________________________________________________________________________________________
global_max_pooling2d_1 (GlobalM (None, 2048)         0           xception[1][0]                   
__________________________________________________________________________________________________
concatenate_1 (Concatenate)     (None, 2648)         0           global_max_pooling2d_2[0][0]     
                                                                 global_max_pooling2d_1[0][0]     
__________________________________________________________________________________________________
alpha_dropout_1 (AlphaDropout)  (None, 2648)         0           concatenate_1[0][0]              
__________________________________________________________________________________________________
pre_out (Dense)                 (None, 1200)         3178800     alpha_dropout_1[0][0]            
__________________________________________________________________________________________________
out (Dense)                     (None, 1010)         1213010     pre_out[0][0]                    
==================================================================================================
Total params: 36,313,090
Trainable params: 36,258,562
Non-trainable params: 54,528
__________________________________________________________________________________________________
In [27]:
uptime=datetime.datetime.now()-date_depart
logging.info("main train %s",uptime)
root        : INFO     main train 3:05:37.701667
In [28]:
hist=model.fit_generator(batch_gen(df_train,batch=full_train_batch_size,augmenters=augmenters,dmax=dmax,dmin=dmin,dcrop=dcrop),
                    steps_per_epoch=1280/full_train_batch_size, epochs=epochs, verbose=1,
                     validation_data=val_gen,
                     validation_steps=400/val_batch_size,
                         callbacks=   callbacks 
                        )
model.save(fichier_modele)
print(datetime.datetime.now()-date_depart)
                    
traincallback: INFO     training start: model_1
Epoch 1/100000000
320/320 [==============================] - 827s 3s/step - loss: 48.1410 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.8948 - mean_anwer_guess: 448.5000 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0180 - val_loss: 12.1783 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9228 - val_mean_anwer_guess: 505.3300 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0000e+00 - val_sparse_top_10_categorical_accuracy: 0.0050
traincallback: INFO     epoch: 0 - val_loss: 12.17833743095398 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.922817440032959 val_mean_anwer_guess: 505.33 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0 val_sparse_top_10_categorical_accuracy: 0.005 loss: 48.14103074967861 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.894803048670292 mean_anwer_guess: 448.5 sparse_top_2_categorical_accuracy: 0.00390625 sparse_top_5_categorical_accuracy: 0.00859375 sparse_top_10_categorical_accuracy: 0.01796875 lr: 0.001 
Epoch 2/100000000
320/320 [==============================] - 816s 3s/step - loss: 12.4288 - sparse_categorical_accuracy: 0.0023 - sparse_categorical_crossentropy: 6.8629 - mean_anwer_guess: 441.7938 - sparse_top_2_categorical_accuracy: 0.0063 - sparse_top_5_categorical_accuracy: 0.0156 - sparse_top_10_categorical_accuracy: 0.0250 - val_loss: 12.6009 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9368 - val_mean_anwer_guess: 506.7000 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0025 - val_sparse_top_10_categorical_accuracy: 0.0075
traincallback: INFO     epoch: 1 - val_loss: 12.600927228927612 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.936796236038208 val_mean_anwer_guess: 506.7 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0025 val_sparse_top_10_categorical_accuracy: 0.0075 loss: 12.428814068436623 sparse_categorical_accuracy: 0.00234375 sparse_categorical_crossentropy: 6.8629228070378305 mean_anwer_guess: 441.79375 sparse_top_2_categorical_accuracy: 0.00625 sparse_top_5_categorical_accuracy: 0.015625 sparse_top_10_categorical_accuracy: 0.025 lr: 0.001 
Epoch 3/100000000
320/320 [==============================] - 819s 3s/step - loss: 12.6151 - sparse_categorical_accuracy: 0.0047 - sparse_categorical_crossentropy: 6.8430 - mean_anwer_guess: 454.4031 - sparse_top_2_categorical_accuracy: 0.0078 - sparse_top_5_categorical_accuracy: 0.0141 - sparse_top_10_categorical_accuracy: 0.0219 - val_loss: 12.6996 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9636 - val_mean_anwer_guess: 509.5800 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0025 - val_sparse_top_10_categorical_accuracy: 0.0025
traincallback: INFO     epoch: 2 - val_loss: 12.69960337638855 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.9635548448562625 val_mean_anwer_guess: 509.58 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0025 val_sparse_top_10_categorical_accuracy: 0.0025 loss: 12.615091639757157 sparse_categorical_accuracy: 0.0046875 sparse_categorical_crossentropy: 6.842999868094921 mean_anwer_guess: 454.403125 sparse_top_2_categorical_accuracy: 0.0078125 sparse_top_5_categorical_accuracy: 0.0140625 sparse_top_10_categorical_accuracy: 0.021875 lr: 0.001 
Epoch 4/100000000
320/320 [==============================] - 798s 2s/step - loss: 12.6081 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.8457 - mean_anwer_guess: 471.2844 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0164 - val_loss: 12.3423 - val_sparse_categorical_accuracy: 0.0025 - val_sparse_categorical_crossentropy: 6.9547 - val_mean_anwer_guess: 486.5700 - val_sparse_top_2_categorical_accuracy: 0.0075 - val_sparse_top_5_categorical_accuracy: 0.0100 - val_sparse_top_10_categorical_accuracy: 0.0125
traincallback: INFO     epoch: 3 - val_loss: 12.342320804595948 val_sparse_categorical_accuracy: 0.0025 val_sparse_categorical_crossentropy: 6.954720191955566 val_mean_anwer_guess: 486.57 val_sparse_top_2_categorical_accuracy: 0.0075 val_sparse_top_5_categorical_accuracy: 0.01 val_sparse_top_10_categorical_accuracy: 0.0125 loss: 12.608069211244583 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.84574968367815 mean_anwer_guess: 471.284375 sparse_top_2_categorical_accuracy: 0.00390625 sparse_top_5_categorical_accuracy: 0.00859375 sparse_top_10_categorical_accuracy: 0.01640625 lr: 0.001 
Epoch 5/100000000
319/320 [============================>.] - ETA: 2s - loss: 12.6846 - sparse_categorical_accuracy: 0.0024 - sparse_categorical_crossentropy: 6.8427 - mean_anwer_guess: 471.7962 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0165
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3030] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3030] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3031] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3031] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3032] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3032] is not in the [index]'
320/320 [==============================] - 807s 3s/step - loss: 12.6841 - sparse_categorical_accuracy: 0.0023 - sparse_categorical_crossentropy: 6.8420 - mean_anwer_guess: 471.2313 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0164 - val_loss: 12.8512 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9653 - val_mean_anwer_guess: 520.8500 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0025 - val_sparse_top_10_categorical_accuracy: 0.0125
traincallback: INFO     epoch: 4 - val_loss: 12.851205167770386 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.9653181552886965 val_mean_anwer_guess: 520.85 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0025 val_sparse_top_10_categorical_accuracy: 0.0125 loss: 12.684120106697083 sparse_categorical_accuracy: 0.00234375 sparse_categorical_crossentropy: 6.842004343867302 mean_anwer_guess: 471.23125 sparse_top_2_categorical_accuracy: 0.00390625 sparse_top_5_categorical_accuracy: 0.00859375 sparse_top_10_categorical_accuracy: 0.01640625 lr: 0.001 
Epoch 6/100000000
320/320 [==============================] - 807s 3s/step - loss: 12.7279 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 6.8252 - mean_anwer_guess: 444.2563 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0063 - sparse_top_10_categorical_accuracy: 0.0141 - val_loss: 12.5491 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9697 - val_mean_anwer_guess: 464.0200 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0050 - val_sparse_top_10_categorical_accuracy: 0.0100

Epoch 00006: val_sparse_categorical_accuracy did not improve from 0.00000
traincallback: INFO     epoch: 5 - val_loss: 12.549130172729493 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.969736785888672 val_mean_anwer_guess: 464.02 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.005 val_sparse_top_10_categorical_accuracy: 0.01 loss: 12.727933862805367 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 6.825203020870686 mean_anwer_guess: 444.25625 sparse_top_2_categorical_accuracy: 0.00390625 sparse_top_5_categorical_accuracy: 0.00625 sparse_top_10_categorical_accuracy: 0.0140625 lr: 0.001 
Epoch 7/100000000
320/320 [==============================] - 804s 3s/step - loss: 12.7585 - sparse_categorical_accuracy: 0.0023 - sparse_categorical_crossentropy: 6.8074 - mean_anwer_guess: 429.9875 - sparse_top_2_categorical_accuracy: 0.0055 - sparse_top_5_categorical_accuracy: 0.0125 - sparse_top_10_categorical_accuracy: 0.0211 - val_loss: 12.7553 - val_sparse_categorical_accuracy: 0.0025 - val_sparse_categorical_crossentropy: 6.9717 - val_mean_anwer_guess: 526.0300 - val_sparse_top_2_categorical_accuracy: 0.0050 - val_sparse_top_5_categorical_accuracy: 0.0100 - val_sparse_top_10_categorical_accuracy: 0.0225
traincallback: INFO     epoch: 6 - val_loss: 12.755282974243164 val_sparse_categorical_accuracy: 0.0025 val_sparse_categorical_crossentropy: 6.97173378944397 val_mean_anwer_guess: 526.03 val_sparse_top_2_categorical_accuracy: 0.005 val_sparse_top_5_categorical_accuracy: 0.01 val_sparse_top_10_categorical_accuracy: 0.0225 loss: 12.758541065454484 sparse_categorical_accuracy: 0.00234375 sparse_categorical_crossentropy: 6.807421277463436 mean_anwer_guess: 429.9875 sparse_top_2_categorical_accuracy: 0.00546875 sparse_top_5_categorical_accuracy: 0.0125 sparse_top_10_categorical_accuracy: 0.02109375 lr: 0.001 
Epoch 8/100000000
320/320 [==============================] - 794s 2s/step - loss: 12.7336 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.8156 - mean_anwer_guess: 451.8687 - sparse_top_2_categorical_accuracy: 0.0016 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0148 - val_loss: 12.7757 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9916 - val_mean_anwer_guess: 541.2100 - val_sparse_top_2_categorical_accuracy: 0.0025 - val_sparse_top_5_categorical_accuracy: 0.0050 - val_sparse_top_10_categorical_accuracy: 0.0100
traincallback: INFO     epoch: 7 - val_loss: 12.775742168426513 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.991552557945251 val_mean_anwer_guess: 541.21 val_sparse_top_2_categorical_accuracy: 0.0025 val_sparse_top_5_categorical_accuracy: 0.005 val_sparse_top_10_categorical_accuracy: 0.01 loss: 12.733552849292755 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.815575960278511 mean_anwer_guess: 451.86875 sparse_top_2_categorical_accuracy: 0.0015625 sparse_top_5_categorical_accuracy: 0.00859375 sparse_top_10_categorical_accuracy: 0.01484375 lr: 0.001 
Epoch 9/100000000
320/320 [==============================] - 786s 2s/step - loss: 12.7661 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 6.8123 - mean_anwer_guess: 441.0469 - sparse_top_2_categorical_accuracy: 0.0023 - sparse_top_5_categorical_accuracy: 0.0055 - sparse_top_10_categorical_accuracy: 0.0156 - val_loss: 12.7747 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9789 - val_mean_anwer_guess: 501.0600 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0000e+00 - val_sparse_top_10_categorical_accuracy: 0.0050
traincallback: INFO     epoch: 8 - val_loss: 12.774747228622436 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.978899388313294 val_mean_anwer_guess: 501.06 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0 val_sparse_top_10_categorical_accuracy: 0.005 loss: 12.766103932261467 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 6.812343089282512 mean_anwer_guess: 441.046875 sparse_top_2_categorical_accuracy: 0.00234375 sparse_top_5_categorical_accuracy: 0.00546875 sparse_top_10_categorical_accuracy: 0.015625 lr: 0.001 
Epoch 10/100000000
320/320 [==============================] - 781s 2s/step - loss: 12.8401 - sparse_categorical_accuracy: 0.0023 - sparse_categorical_crossentropy: 6.8295 - mean_anwer_guess: 439.9531 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0078 - sparse_top_10_categorical_accuracy: 0.0164 - val_loss: 12.8246 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9870 - val_mean_anwer_guess: 507.5800 - val_sparse_top_2_categorical_accuracy: 0.0025 - val_sparse_top_5_categorical_accuracy: 0.0050 - val_sparse_top_10_categorical_accuracy: 0.0075
traincallback: INFO     epoch: 9 - val_loss: 12.824644756317138 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.986988925933838 val_mean_anwer_guess: 507.58 val_sparse_top_2_categorical_accuracy: 0.0025 val_sparse_top_5_categorical_accuracy: 0.005 val_sparse_top_10_categorical_accuracy: 0.0075 loss: 12.840075600147248 sparse_categorical_accuracy: 0.00234375 sparse_categorical_crossentropy: 6.829490113258362 mean_anwer_guess: 439.953125 sparse_top_2_categorical_accuracy: 0.00390625 sparse_top_5_categorical_accuracy: 0.0078125 sparse_top_10_categorical_accuracy: 0.01640625 lr: 0.001 
Epoch 11/100000000
320/320 [==============================] - 769s 2s/step - loss: 12.7090 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.7856 - mean_anwer_guess: 438.3531 - sparse_top_2_categorical_accuracy: 0.0023 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0172 - val_loss: 12.6598 - val_sparse_categorical_accuracy: 0.0025 - val_sparse_categorical_crossentropy: 6.9976 - val_mean_anwer_guess: 535.0700 - val_sparse_top_2_categorical_accuracy: 0.0025 - val_sparse_top_5_categorical_accuracy: 0.0050 - val_sparse_top_10_categorical_accuracy: 0.0075
traincallback: INFO     epoch: 10 - val_loss: 12.659755830764771 val_sparse_categorical_accuracy: 0.0025 val_sparse_categorical_crossentropy: 6.997620878219604 val_mean_anwer_guess: 535.07 val_sparse_top_2_categorical_accuracy: 0.0025 val_sparse_top_5_categorical_accuracy: 0.005 val_sparse_top_10_categorical_accuracy: 0.0075 loss: 12.708958914875984 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.785642342269421 mean_anwer_guess: 438.353125 sparse_top_2_categorical_accuracy: 0.00234375 sparse_top_5_categorical_accuracy: 0.00859375 sparse_top_10_categorical_accuracy: 0.0171875 lr: 0.001 
Epoch 12/100000000
320/320 [==============================] - 783s 2s/step - loss: 12.7437 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 6.7799 - mean_anwer_guess: 440.8656 - sparse_top_2_categorical_accuracy: 0.0016 - sparse_top_5_categorical_accuracy: 0.0070 - sparse_top_10_categorical_accuracy: 0.0148 - val_loss: 12.8805 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 7.0201 - val_mean_anwer_guess: 497.7000 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0050 - val_sparse_top_10_categorical_accuracy: 0.0100
traincallback: INFO     epoch: 11 - val_loss: 12.880471305847168 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 7.020058689117431 val_mean_anwer_guess: 497.7 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.005 val_sparse_top_10_categorical_accuracy: 0.01 loss: 12.743705308437347 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 6.77988765835762 mean_anwer_guess: 440.865625 sparse_top_2_categorical_accuracy: 0.0015625 sparse_top_5_categorical_accuracy: 0.00703125 sparse_top_10_categorical_accuracy: 0.01484375 lr: 0.001 
Epoch 13/100000000
319/320 [============================>.] - ETA: 2s - loss: 12.8122 - sparse_categorical_accuracy: 7.8370e-04 - sparse_categorical_crossentropy: 6.7959 - mean_anwer_guess: 490.0690 - sparse_top_2_categorical_accuracy: 0.0016 - sparse_top_5_categorical_accuracy: 0.0063 - sparse_top_10_categorical_accuracy: 0.0172
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3030] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3030] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3031] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3031] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3032] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3032] is not in the [index]'
320/320 [==============================] - 769s 2s/step - loss: 12.8128 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 6.7966 - mean_anwer_guess: 490.5000 - sparse_top_2_categorical_accuracy: 0.0016 - sparse_top_5_categorical_accuracy: 0.0063 - sparse_top_10_categorical_accuracy: 0.0172 - val_loss: 12.9069 - val_sparse_categorical_accuracy: 0.0025 - val_sparse_categorical_crossentropy: 6.9953 - val_mean_anwer_guess: 468.6600 - val_sparse_top_2_categorical_accuracy: 0.0050 - val_sparse_top_5_categorical_accuracy: 0.0125 - val_sparse_top_10_categorical_accuracy: 0.0175
traincallback: INFO     epoch: 12 - val_loss: 12.906928291320801 val_sparse_categorical_accuracy: 0.0025 val_sparse_categorical_crossentropy: 6.995328159332275 val_mean_anwer_guess: 468.66 val_sparse_top_2_categorical_accuracy: 0.005 val_sparse_top_5_categorical_accuracy: 0.0125 val_sparse_top_10_categorical_accuracy: 0.0175 loss: 12.812796035408974 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 6.796648205816746 mean_anwer_guess: 490.5 sparse_top_2_categorical_accuracy: 0.0015625 sparse_top_5_categorical_accuracy: 0.00625 sparse_top_10_categorical_accuracy: 0.0171875 lr: 0.001 
Epoch 14/100000000
320/320 [==============================] - 796s 2s/step - loss: 12.8612 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 6.7987 - mean_anwer_guess: 458.7531 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0070 - sparse_top_10_categorical_accuracy: 0.0172 - val_loss: 13.0151 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 7.0287 - val_mean_anwer_guess: 462.7200 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0025 - val_sparse_top_10_categorical_accuracy: 0.0075
traincallback: INFO     epoch: 13 - val_loss: 13.015073108673096 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 7.028693838119507 val_mean_anwer_guess: 462.72 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0025 val_sparse_top_10_categorical_accuracy: 0.0075 loss: 12.86115721464157 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 6.798652696609497 mean_anwer_guess: 458.753125 sparse_top_2_categorical_accuracy: 0.00390625 sparse_top_5_categorical_accuracy: 0.00703125 sparse_top_10_categorical_accuracy: 0.0171875 lr: 0.001 
Epoch 00014: ReduceLROnPlateau reducing learning rate to 0.0006000000284984708.
Epoch 15/100000000
320/320 [==============================] - 795s 2s/step - loss: 10.4306 - sparse_categorical_accuracy: 0.0023 - sparse_categorical_crossentropy: 6.7886 - mean_anwer_guess: 453.1719 - sparse_top_2_categorical_accuracy: 0.0031 - sparse_top_5_categorical_accuracy: 0.0055 - sparse_top_10_categorical_accuracy: 0.0211 - val_loss: 10.2737 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9637 - val_mean_anwer_guess: 490.0400 - val_sparse_top_2_categorical_accuracy: 0.0025 - val_sparse_top_5_categorical_accuracy: 0.0025 - val_sparse_top_10_categorical_accuracy: 0.0175
traincallback: INFO     epoch: 14 - val_loss: 10.273672542572022 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.963716921806335 val_mean_anwer_guess: 490.04 val_sparse_top_2_categorical_accuracy: 0.0025 val_sparse_top_5_categorical_accuracy: 0.0025 val_sparse_top_10_categorical_accuracy: 0.0175 loss: 10.430625811219215 sparse_categorical_accuracy: 0.00234375 sparse_categorical_crossentropy: 6.788645024597645 mean_anwer_guess: 453.171875 sparse_top_2_categorical_accuracy: 0.003125 sparse_top_5_categorical_accuracy: 0.00546875 sparse_top_10_categorical_accuracy: 0.02109375 lr: 0.0006 
Epoch 16/100000000
320/320 [==============================] - 784s 2s/step - loss: 10.3938 - sparse_categorical_accuracy: 0.0023 - sparse_categorical_crossentropy: 6.7822 - mean_anwer_guess: 432.9562 - sparse_top_2_categorical_accuracy: 0.0047 - sparse_top_5_categorical_accuracy: 0.0070 - sparse_top_10_categorical_accuracy: 0.0148 - val_loss: 10.3957 - val_sparse_categorical_accuracy: 0.0025 - val_sparse_categorical_crossentropy: 6.9853 - val_mean_anwer_guess: 480.5500 - val_sparse_top_2_categorical_accuracy: 0.0075 - val_sparse_top_5_categorical_accuracy: 0.0075 - val_sparse_top_10_categorical_accuracy: 0.0175
traincallback: INFO     epoch: 15 - val_loss: 10.395728225708007 val_sparse_categorical_accuracy: 0.0025 val_sparse_categorical_crossentropy: 6.98533519744873 val_mean_anwer_guess: 480.55 val_sparse_top_2_categorical_accuracy: 0.0075 val_sparse_top_5_categorical_accuracy: 0.0075 val_sparse_top_10_categorical_accuracy: 0.0175 loss: 10.393761798739433 sparse_categorical_accuracy: 0.00234375 sparse_categorical_crossentropy: 6.782198789715767 mean_anwer_guess: 432.95625 sparse_top_2_categorical_accuracy: 0.0046875 sparse_top_5_categorical_accuracy: 0.00703125 sparse_top_10_categorical_accuracy: 0.01484375 lr: 0.0006 
Epoch 17/100000000
320/320 [==============================] - 795s 2s/step - loss: 10.3923 - sparse_categorical_accuracy: 0.0031 - sparse_categorical_crossentropy: 6.7879 - mean_anwer_guess: 436.1344 - sparse_top_2_categorical_accuracy: 0.0031 - sparse_top_5_categorical_accuracy: 0.0117 - sparse_top_10_categorical_accuracy: 0.0172 - val_loss: 10.5543 - val_sparse_categorical_accuracy: 0.0025 - val_sparse_categorical_crossentropy: 6.9715 - val_mean_anwer_guess: 510.8900 - val_sparse_top_2_categorical_accuracy: 0.0025 - val_sparse_top_5_categorical_accuracy: 0.0025 - val_sparse_top_10_categorical_accuracy: 0.0050
traincallback: INFO     epoch: 16 - val_loss: 10.554329509735107 val_sparse_categorical_accuracy: 0.0025 val_sparse_categorical_crossentropy: 6.971497311592102 val_mean_anwer_guess: 510.89 val_sparse_top_2_categorical_accuracy: 0.0025 val_sparse_top_5_categorical_accuracy: 0.0025 val_sparse_top_10_categorical_accuracy: 0.005 loss: 10.392290279269218 sparse_categorical_accuracy: 0.003125 sparse_categorical_crossentropy: 6.787858499586582 mean_anwer_guess: 436.134375 sparse_top_2_categorical_accuracy: 0.003125 sparse_top_5_categorical_accuracy: 0.01171875 sparse_top_10_categorical_accuracy: 0.0171875 lr: 0.0006 
Epoch 18/100000000
320/320 [==============================] - 785s 2s/step - loss: 10.3798 - sparse_categorical_accuracy: 0.0039 - sparse_categorical_crossentropy: 6.7880 - mean_anwer_guess: 453.6187 - sparse_top_2_categorical_accuracy: 0.0078 - sparse_top_5_categorical_accuracy: 0.0117 - sparse_top_10_categorical_accuracy: 0.0187 - val_loss: 10.6460 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9940 - val_mean_anwer_guess: 533.2100 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0050 - val_sparse_top_10_categorical_accuracy: 0.0100
root        : INFO     end date
traincallback: INFO     epoch: 17 - val_loss: 10.646020088195801 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.994029064178466 val_mean_anwer_guess: 533.21 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.005 val_sparse_top_10_categorical_accuracy: 0.01 loss: 10.379760584235191 sparse_categorical_accuracy: 0.00390625 sparse_categorical_crossentropy: 6.788010680675507 mean_anwer_guess: 453.61875 sparse_top_2_categorical_accuracy: 0.0078125 sparse_top_5_categorical_accuracy: 0.01171875 sparse_top_10_categorical_accuracy: 0.01875 lr: 0.0006 
traincallback: INFO     training end: model_1
7:04:41.866862
In [29]:
uptime=datetime.datetime.now()-date_depart
logging.info("post train %s",uptime)
logging.info("remaining time %s",datetime.timedelta(hours=9)+date_depart-datetime.datetime.now())
root        : INFO     post train 7:04:44.719416
root        : INFO     remaining time 1:55:15.274879
In [30]:
if (datetime.datetime.now()-date_depart)<datetime.timedelta(hours=8,minutes=30):
    fig=plt.figure(figsize=(15,7))
    plt.subplot("211")
    train_history=hist.history
    histories=[hist_pre.history,hist.history,hist_post]
    for k in train_history.keys():
        train_history[k]=[]
        for h in histories:
            train_history[k]+=h.get(k,[])


    for k in train_history.keys():
        if "acc" in k:
            plt.plot(train_history[k],label=k)
    plt.ylim(ymin=0.95,ymax=1.0)
    plt.legend()
    plt.subplot("212")
    plt.yscale("log")
    for k in train_history.keys():
        if "loss" in k:
            plt.plot(train_history[k],label=k)
    plt.legend()

    plt.ylim(ymax=0.8)
    fig.savefig("graph.png",dpi=200,transparent=False)
    #IPython.display.Image(filename="graph.png")
---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-30-6e6b8d3c5db1> in <module>()
      3     plt.subplot("211")
      4     train_history=hist.history
----> 5     histories=[hist_pre.history,hist.history,hist_post]
      6     for k in train_history.keys():
      7         train_history[k]=[]

NameError: name 'hist_post' is not defined
In [31]:
if (datetime.datetime.now()-date_depart)<datetime.timedelta(hours=7,minutes=30):
    for m,e in zip(model.metrics_names,
                   model.evaluate_generator(batch_gen(df_val,batch=64),steps=50, verbose=1) ):
                   print (m,e)
45/50 [==========================>...] - ETA: 12s
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3030] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3030] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3031] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3031] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3032] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3032] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3033] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3033] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3034] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3034] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3035] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3035] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3036] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3036] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3037] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3037] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3038] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3038] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3039] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3039] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3040] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3040] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3041] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3041] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3042] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3042] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3043] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3043] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3044] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3044] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3045] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3045] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3046] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3046] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3047] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3047] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3048] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3048] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3049] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3049] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3050] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3050] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3051] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3051] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3052] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3052] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3053] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3053] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3054] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3054] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3055] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3055] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3056] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3056] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3057] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3057] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3058] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3058] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3059] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3059] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3060] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3060] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3061] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3061] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3062] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3062] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3063] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3063] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3064] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3064] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3065] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3065] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3066] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3066] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3067] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3067] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3068] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3068] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3069] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3069] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3070] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3070] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3071] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3071] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3072] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3072] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3073] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3073] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3074] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3074] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3075] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3075] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3076] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3076] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3077] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3077] is not in the [index]'
root        : ERROR    imgaug_batch_gen
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3078] is not in the [index]'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen
    if df.loc[j+i,"req"] is None:
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__
    return self._getitem_tuple(key)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple
    return self._getitem_lowerdim(tup)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim
    section = self._getitem_axis(key, axis=i)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis
    self._validate_key(key, axis)
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key
    error()
  File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error
    axis=self.obj._get_axis_name(axis)))
KeyError: 'the label [3078] is not in the [index]'
50/50 [==============================] - 127s 3s/step
loss 12.688553867340088
sparse_categorical_accuracy 0.0009375
sparse_categorical_crossentropy 7.004177312850953
mean_anwer_guess 465.0
sparse_top_2_categorical_accuracy 0.001875
sparse_top_5_categorical_accuracy 0.004375
sparse_top_10_categorical_accuracy 0.0090625
In [32]:
if (datetime.datetime.now()-date_depart)<datetime.timedelta(hours=7,minutes=30):

    ann_file = '../input/inaturalist-2019-fgvc6/test2019.json'
    with open(ann_file) as data_file:
            test_anns = json.load(data_file)



    df_test=pd.DataFrame(test_anns['images'])[["id","file_name"]]
    df_test
In [33]:
test_batch=50

if (datetime.datetime.now()-date_depart)<datetime.timedelta(hours=7,minutes=0): 
    test_datagen = ImageDataGenerator(rescale=1./255)
    test_generator = test_datagen.flow_from_dataframe(      

            dataframe=df_test,    

            directory = "../input/inaturalist-2019-fgvc6/test2019",    
            x_col="file_name",
            target_size = (dcrop,dcrop),
            batch_size = test_batch,
            shuffle = False,
            class_mode = None
            )
    lengen=len(test_generator.filenames)
    #math.gcd(lengen)
    predict=model.predict_generator(test_generator, steps = len(test_generator.filenames)//test_batch+1,verbose=1)
    sub=np.argsort(predict)[:lengen,-10:]
    sub=np.flip(sub,1)
    df_test["predicted"]=[" ".join(str(n) for n in  pred  )for pred in sub  ]
    df_test["preds"]=df_test["predicted"]
   
    df_test[["id","preds", 'predicted']].to_csv("submission.csv", index=False)
    df_test
    
In [34]:
uptime=datetime.datetime.now()-date_depart
logging.info("end %s",uptime)
logging.info("remaining time %s",datetime.timedelta(hours=9)+date_depart-datetime.datetime.now())
root        : INFO     end 7:06:57.295425
root        : INFO     remaining time 1:53:02.703319