From: https://www.kaggle.com/cedriclacrambe/inaturalist-xception-512
Author: Cédric Lacrambe
# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load in
import datetime
date_depart=datetime.datetime.now()
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import weakref
import warnings
warnings.simplefilter("ignore")
import logging
logging.basicConfig(filename='python.log',level=logging.DEBUG)
logging.captureWarnings(True)
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
# tell the handler to use this format
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
from functools import partial
# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory
import cv2
import matplotlib.pyplot as plt
import imageio
import imgaug
from imgaug import augmenters as iaa
from imgaug import parameters as iap
import keras
from keras.models import Sequential, Model
from keras.layers import Dense, Flatten, Activation, Dropout, GlobalAveragePooling2D
from keras.preprocessing.image import ImageDataGenerator
from keras import optimizers, applications
from keras.callbacks import ModelCheckpoint, LearningRateScheduler, TensorBoard, EarlyStopping
from keras import backend as K
from concurrent import futures
import os
import json
print(os.listdir("../input"))
# Any results you write to the current directory are saved as output.
Using TensorFlow backend.
['inaturalist-2019-fgvc6', 'xception', 'inaturalist-xception-512']
#!ls -Rlh ../input
duree_max=datetime.timedelta(hours=7,minutes=30)
fichier_modele_base="inaturalist"
train_batch_size=64
full_train_batch_size=4
val_batch_size=4
epochs=int(1e8)
load_keras_weights=False
dmax=1024
dmin=600
dcrop=512
date_limite= date_depart+duree_max
ann_file = '../input/inaturalist-2019-fgvc6/train2019.json'
with open(ann_file) as data_file:
train_anns = json.load(data_file)
def get_file_list(ann_file):
with open(ann_file) as data_file:
train_anns = json.load(data_file)
train_anns_df = pd.DataFrame(train_anns['annotations'])[['image_id','category_id']]
train_img_df = pd.DataFrame(train_anns['images'])[['id', 'file_name']].rename(columns={'id':'image_id'})
train_anns_df [train_anns_df.image_id.duplicated()]
df_train=pd.merge(train_img_df,train_anns_df)[['category_id', 'file_name']]
return df_train
df_train=get_file_list(ann_file)
df_train
category_id | file_name | |
---|---|---|
0 | 400 | train_val2019/Plants/400/d1322d13ccd856eb4236c... |
1 | 570 | train_val2019/Plants/570/15edbc1e2ef000d8ace48... |
2 | 167 | train_val2019/Reptiles/167/c87a32e8927cbf4f06d... |
3 | 254 | train_val2019/Birds/254/9fcdd1d37e96d8fd94dfdc... |
4 | 739 | train_val2019/Plants/739/ffa06f951e99de9d220ae... |
5 | 246 | train_val2019/Birds/246/ef5109f4c0bd04ca509fad... |
6 | 25 | train_val2019/Insects/25/5ee1691712dffd9e3a178... |
7 | 386 | train_val2019/Plants/386/3af7b0ad2e2bc118e38c5... |
8 | 515 | train_val2019/Plants/515/4bfd750cbb32bb0e302ca... |
9 | 259 | train_val2019/Birds/259/353bd26bf9e5e1606a9189... |
10 | 824 | train_val2019/Plants/824/2543714c2f1a8611f1a78... |
11 | 234 | train_val2019/Birds/234/dc64a999cacea6eb5ed512... |
12 | 363 | train_val2019/Plants/363/aec09afe1abc6aa7dc70c... |
13 | 235 | train_val2019/Birds/235/73d440f2157a161fec3e77... |
14 | 862 | train_val2019/Plants/862/e4d28d6347aa0d38322a6... |
15 | 731 | train_val2019/Plants/731/032b0860dcd8ae04aaab4... |
16 | 325 | train_val2019/Birds/325/c1db84df7d22e0933df6b6... |
17 | 213 | train_val2019/Birds/213/091a60ad7d3cbec3141eea... |
18 | 149 | train_val2019/Insects/149/2cd40eee1996f0eb3187... |
19 | 131 | train_val2019/Insects/131/6e84516b34cdc45be2a3... |
20 | 167 | train_val2019/Reptiles/167/e21448daf063c25cb56... |
21 | 151 | train_val2019/Insects/151/c84bc5a5187096098f4f... |
22 | 751 | train_val2019/Plants/751/c3fb975e1120f46671632... |
23 | 270 | train_val2019/Birds/270/8d00e0ba26503ec41aa884... |
24 | 205 | train_val2019/Birds/205/430ba5e10dc2a2888f8b14... |
25 | 415 | train_val2019/Plants/415/c49a1cd3a1d83a15e9274... |
26 | 149 | train_val2019/Insects/149/e0587065186b2bd34478... |
27 | 739 | train_val2019/Plants/739/16494b1e8cdcc0cbf5c13... |
28 | 8 | train_val2019/Fungi/8/ceea3c6d5ff90364486f5683... |
29 | 922 | train_val2019/Plants/922/9635afacfa8e46e0390c2... |
... | ... | ... |
265183 | 604 | train_val2019/Plants/604/32958abfc2eb70b849356... |
265184 | 236 | train_val2019/Birds/236/e2197c5fa669685dc45221... |
265185 | 660 | train_val2019/Plants/660/6300ce1be902e9c5ca97e... |
265186 | 353 | train_val2019/Plants/353/39882460221ea4acf9879... |
265187 | 387 | train_val2019/Plants/387/e9060a2a376ef9d53d55e... |
265188 | 907 | train_val2019/Plants/907/45503a52d6ff851ee2aac... |
265189 | 968 | train_val2019/Plants/968/0ee365c29cbf8e328ccb1... |
265190 | 188 | train_val2019/Reptiles/188/128f298609594ba84cc... |
265191 | 153 | train_val2019/Amphibians/153/f1e8b2d3b668b9922... |
265192 | 970 | train_val2019/Plants/970/8fd903f36f2132605d3bb... |
265193 | 350 | train_val2019/Plants/350/c60f16cb442d9f40e11eb... |
265194 | 712 | train_val2019/Plants/712/649689c72339cd76f0220... |
265195 | 526 | train_val2019/Plants/526/feb980e3cad11fbf56940... |
265196 | 233 | train_val2019/Birds/233/c06b3e72c9796fc266969c... |
265197 | 328 | train_val2019/Plants/328/65bfb2a6fe3dceb3e0b02... |
265198 | 211 | train_val2019/Birds/211/f1f83215f9cdbc29064b9e... |
265199 | 268 | train_val2019/Birds/268/6f303db38e86c4dd5ba943... |
265200 | 321 | train_val2019/Birds/321/540a4789c221ca78be9a30... |
265201 | 273 | train_val2019/Birds/273/335c36a57fbd52e6071c4e... |
265202 | 477 | train_val2019/Plants/477/04234ec77dd9d48512a6d... |
265203 | 420 | train_val2019/Plants/420/5802a39c2f868ffa4ad6e... |
265204 | 77 | train_val2019/Insects/77/41f0485a9fc909328b8a5... |
265205 | 1001 | train_val2019/Plants/1001/c1813d4501453affaeaf... |
265206 | 375 | train_val2019/Plants/375/d4b9b81094127add57afe... |
265207 | 532 | train_val2019/Plants/532/19b8c68c716998290f242... |
265208 | 223 | train_val2019/Birds/223/ebf4f99d55fb0071c49e08... |
265209 | 188 | train_val2019/Reptiles/188/cb0600062644540aaf2... |
265210 | 70 | train_val2019/Insects/70/bfd566f9ae5177f5f2b52... |
265211 | 587 | train_val2019/Plants/587/59c3e60d021606bf352de... |
265212 | 287 | train_val2019/Birds/287/99da04ba30e67391f9a35b... |
265213 rows × 2 columns
df_val=get_file_list('../input/inaturalist-2019-fgvc6/val2019.json')
df_val
category_id | file_name | |
---|---|---|
0 | 644 | train_val2019/Plants/644/716a69838526f3ada3b2f... |
1 | 597 | train_val2019/Plants/597/0942cc64d2e759c5ee059... |
2 | 883 | train_val2019/Plants/883/acfdbfd9fa675f1c84558... |
3 | 300 | train_val2019/Birds/300/5f3194ff536c7dd31d80b7... |
4 | 881 | train_val2019/Plants/881/76acaf0b2841f91982d21... |
5 | 771 | train_val2019/Plants/771/5d190fd90da893988a3c9... |
6 | 607 | train_val2019/Plants/607/8fb0ddfe92dadbf9c5753... |
7 | 698 | train_val2019/Plants/698/6d71521a64d1e2fe8bb34... |
8 | 714 | train_val2019/Plants/714/09508f2bf937d21d63297... |
9 | 316 | train_val2019/Birds/316/21c700bc90523485af6730... |
10 | 726 | train_val2019/Plants/726/847b2ebf8efc1a528c2d3... |
11 | 27 | train_val2019/Insects/27/e5d141185a3a50b544d15... |
12 | 640 | train_val2019/Plants/640/cbe5ff1a159b614ae8677... |
13 | 756 | train_val2019/Plants/756/a99c33a5f954ffecd7340... |
14 | 73 | train_val2019/Insects/73/2fc6b41247af765af9984... |
15 | 594 | train_val2019/Plants/594/d98519788816a0f61bd15... |
16 | 179 | train_val2019/Reptiles/179/914aab5662e77a128c7... |
17 | 440 | train_val2019/Plants/440/02f1f67068df3d8855467... |
18 | 859 | train_val2019/Plants/859/23ba5d0baf198007ec097... |
19 | 5 | train_val2019/Fungi/5/3308225981fc58e6eb6a7889... |
20 | 401 | train_val2019/Plants/401/6c73829cb13a02a69123e... |
21 | 928 | train_val2019/Plants/928/d65f36eaf70136e1d4c2e... |
22 | 216 | train_val2019/Birds/216/f5902f2a9566038b67ff72... |
23 | 200 | train_val2019/Reptiles/200/ef2471af2f01a252250... |
24 | 983 | train_val2019/Plants/983/5b1e6397e48bec711b3a3... |
25 | 350 | train_val2019/Plants/350/8ebad1a2af78183b35bac... |
26 | 223 | train_val2019/Birds/223/e951479ab2f49714025c1e... |
27 | 438 | train_val2019/Plants/438/f94013bba01de6a654f81... |
28 | 834 | train_val2019/Plants/834/4c3d81073f661f93c20c3... |
29 | 248 | train_val2019/Birds/248/0a2da2111a989327d9935b... |
... | ... | ... |
3000 | 681 | train_val2019/Plants/681/3458385697acabf822006... |
3001 | 830 | train_val2019/Plants/830/9a30a84d57d2aa127298e... |
3002 | 60 | train_val2019/Insects/60/cb0032132cbad87f49115... |
3003 | 784 | train_val2019/Plants/784/0cadfce23ea5c5aaff90b... |
3004 | 873 | train_val2019/Plants/873/b6a4bebe95e25647b50fd... |
3005 | 868 | train_val2019/Plants/868/7e6410f5ea26657666206... |
3006 | 704 | train_val2019/Plants/704/e226a1d3713feaafc0443... |
3007 | 617 | train_val2019/Plants/617/dbea4253dbcfb3b06b3c1... |
3008 | 631 | train_val2019/Plants/631/191450a32f3105071caf2... |
3009 | 548 | train_val2019/Plants/548/edd75e1d3fec72691b046... |
3010 | 424 | train_val2019/Plants/424/047f1de7a61605cb15b5a... |
3011 | 14 | train_val2019/Insects/14/d1d2163a1b64882cd2a7c... |
3012 | 976 | train_val2019/Plants/976/963055c5e9726e06ae73c... |
3013 | 321 | train_val2019/Birds/321/98713839fa00a49b75dc5e... |
3014 | 391 | train_val2019/Plants/391/7628608962c27c640c38d... |
3015 | 862 | train_val2019/Plants/862/f9969d436af626938b9dc... |
3016 | 809 | train_val2019/Plants/809/fe521339f28d17599928c... |
3017 | 813 | train_val2019/Plants/813/275660d70fe22e5ddc0c0... |
3018 | 219 | train_val2019/Birds/219/99e8afe389a7b46fefdb86... |
3019 | 292 | train_val2019/Birds/292/5a1af52f0434e3996f7a5a... |
3020 | 155 | train_val2019/Amphibians/155/009611cb3d44f7a9c... |
3021 | 772 | train_val2019/Plants/772/d83e9fd8c2e3493df4e49... |
3022 | 968 | train_val2019/Plants/968/59d8da88969576e7bae10... |
3023 | 893 | train_val2019/Plants/893/627c72786a18242784b7f... |
3024 | 157 | train_val2019/Amphibians/157/6d3b4f10fa385dd81... |
3025 | 468 | train_val2019/Plants/468/3260e4346ba127a28b564... |
3026 | 1006 | train_val2019/Plants/1006/3b33c0e1fb866c452f01... |
3027 | 472 | train_val2019/Plants/472/44a19d7731dccd92e4c0e... |
3028 | 192 | train_val2019/Reptiles/192/c5f142b44904508c866... |
3029 | 835 | train_val2019/Plants/835/a6f21e1829ff47c54d474... |
3030 rows × 2 columns
train_anns.keys()
dict_keys(['info', 'images', 'licenses', 'annotations', 'categories'])
len(df_train.category_id.unique())
1010
df_valid=get_file_list('../input/inaturalist-2019-fgvc6/train2019.json')
df_valid
category_id | file_name | |
---|---|---|
0 | 400 | train_val2019/Plants/400/d1322d13ccd856eb4236c... |
1 | 570 | train_val2019/Plants/570/15edbc1e2ef000d8ace48... |
2 | 167 | train_val2019/Reptiles/167/c87a32e8927cbf4f06d... |
3 | 254 | train_val2019/Birds/254/9fcdd1d37e96d8fd94dfdc... |
4 | 739 | train_val2019/Plants/739/ffa06f951e99de9d220ae... |
5 | 246 | train_val2019/Birds/246/ef5109f4c0bd04ca509fad... |
6 | 25 | train_val2019/Insects/25/5ee1691712dffd9e3a178... |
7 | 386 | train_val2019/Plants/386/3af7b0ad2e2bc118e38c5... |
8 | 515 | train_val2019/Plants/515/4bfd750cbb32bb0e302ca... |
9 | 259 | train_val2019/Birds/259/353bd26bf9e5e1606a9189... |
10 | 824 | train_val2019/Plants/824/2543714c2f1a8611f1a78... |
11 | 234 | train_val2019/Birds/234/dc64a999cacea6eb5ed512... |
12 | 363 | train_val2019/Plants/363/aec09afe1abc6aa7dc70c... |
13 | 235 | train_val2019/Birds/235/73d440f2157a161fec3e77... |
14 | 862 | train_val2019/Plants/862/e4d28d6347aa0d38322a6... |
15 | 731 | train_val2019/Plants/731/032b0860dcd8ae04aaab4... |
16 | 325 | train_val2019/Birds/325/c1db84df7d22e0933df6b6... |
17 | 213 | train_val2019/Birds/213/091a60ad7d3cbec3141eea... |
18 | 149 | train_val2019/Insects/149/2cd40eee1996f0eb3187... |
19 | 131 | train_val2019/Insects/131/6e84516b34cdc45be2a3... |
20 | 167 | train_val2019/Reptiles/167/e21448daf063c25cb56... |
21 | 151 | train_val2019/Insects/151/c84bc5a5187096098f4f... |
22 | 751 | train_val2019/Plants/751/c3fb975e1120f46671632... |
23 | 270 | train_val2019/Birds/270/8d00e0ba26503ec41aa884... |
24 | 205 | train_val2019/Birds/205/430ba5e10dc2a2888f8b14... |
25 | 415 | train_val2019/Plants/415/c49a1cd3a1d83a15e9274... |
26 | 149 | train_val2019/Insects/149/e0587065186b2bd34478... |
27 | 739 | train_val2019/Plants/739/16494b1e8cdcc0cbf5c13... |
28 | 8 | train_val2019/Fungi/8/ceea3c6d5ff90364486f5683... |
29 | 922 | train_val2019/Plants/922/9635afacfa8e46e0390c2... |
... | ... | ... |
265183 | 604 | train_val2019/Plants/604/32958abfc2eb70b849356... |
265184 | 236 | train_val2019/Birds/236/e2197c5fa669685dc45221... |
265185 | 660 | train_val2019/Plants/660/6300ce1be902e9c5ca97e... |
265186 | 353 | train_val2019/Plants/353/39882460221ea4acf9879... |
265187 | 387 | train_val2019/Plants/387/e9060a2a376ef9d53d55e... |
265188 | 907 | train_val2019/Plants/907/45503a52d6ff851ee2aac... |
265189 | 968 | train_val2019/Plants/968/0ee365c29cbf8e328ccb1... |
265190 | 188 | train_val2019/Reptiles/188/128f298609594ba84cc... |
265191 | 153 | train_val2019/Amphibians/153/f1e8b2d3b668b9922... |
265192 | 970 | train_val2019/Plants/970/8fd903f36f2132605d3bb... |
265193 | 350 | train_val2019/Plants/350/c60f16cb442d9f40e11eb... |
265194 | 712 | train_val2019/Plants/712/649689c72339cd76f0220... |
265195 | 526 | train_val2019/Plants/526/feb980e3cad11fbf56940... |
265196 | 233 | train_val2019/Birds/233/c06b3e72c9796fc266969c... |
265197 | 328 | train_val2019/Plants/328/65bfb2a6fe3dceb3e0b02... |
265198 | 211 | train_val2019/Birds/211/f1f83215f9cdbc29064b9e... |
265199 | 268 | train_val2019/Birds/268/6f303db38e86c4dd5ba943... |
265200 | 321 | train_val2019/Birds/321/540a4789c221ca78be9a30... |
265201 | 273 | train_val2019/Birds/273/335c36a57fbd52e6071c4e... |
265202 | 477 | train_val2019/Plants/477/04234ec77dd9d48512a6d... |
265203 | 420 | train_val2019/Plants/420/5802a39c2f868ffa4ad6e... |
265204 | 77 | train_val2019/Insects/77/41f0485a9fc909328b8a5... |
265205 | 1001 | train_val2019/Plants/1001/c1813d4501453affaeaf... |
265206 | 375 | train_val2019/Plants/375/d4b9b81094127add57afe... |
265207 | 532 | train_val2019/Plants/532/19b8c68c716998290f242... |
265208 | 223 | train_val2019/Birds/223/ebf4f99d55fb0071c49e08... |
265209 | 188 | train_val2019/Reptiles/188/cb0600062644540aaf2... |
265210 | 70 | train_val2019/Insects/70/bfd566f9ae5177f5f2b52... |
265211 | 587 | train_val2019/Plants/587/59c3e60d021606bf352de... |
265212 | 287 | train_val2019/Birds/287/99da04ba30e67391f9a35b... |
265213 rows × 2 columns
classes=len(df_valid.category_id.unique())
print("random accuracy:",1/classes)
classes
random accuracy: 0.0009900990099009901
1010
df_train["catstr"]=df_train.category_id.astype("str")
p=next(df_train.sample(n=5).itertuples())
p.category_id
p.file_name
'train_val2019/Birds/246/ee8c33c450ac2243493896decff953be.jpg'
plt.figure(figsize=(25,25))
for n,tu in enumerate(df_train.sample(n=5).itertuples()):
cat=tu.category_id
im=tu.file_name
plt.subplot(1,5,n+1)
im=os.path.join("../input/inaturalist-2019-fgvc6/train_val2019/",im)
plt.axis("off")
plt.title(cat)
plt.imshow(imageio.imread(im))
augmenters=[iaa.Sometimes(0.5,
[iaa.Affine(scale=(0.99,1.05),
translate_percent=(0,0.05),
rotate=iap.Normal(0,3),
shear=iap.Normal(0,3),
order=3)]),
iaa.Sometimes(0.3,[iaa.PiecewiseAffine(scale=(0,0.02))]),
iaa.Sometimes(0.1,
[iaa.GaussianBlur(sigma=(0, 0.5)) ]),
iaa.Sometimes(0.1,
[iaa.AverageBlur(k=(1, 2))]),
iaa.AdditiveGaussianNoise(scale=(0, 0.05*255)),
iaa.AddElementwise((-5, 5)),
iaa.Sometimes(0.1,
[iaa.Superpixels(p_replace=(0.05, 0.8), n_segments=(16, 128))]
),
iaa.Sometimes(0.3,
[iaa.ElasticTransformation(alpha=(0, 5.0), sigma=(0.1,0.6) )]
),
iaa.SaltAndPepper(p=(0.005,0.1)),
iaa.Sometimes(0.4,
[iaa.CoarseDropout(p=(0, 0.3),size_percent=(0.02, 0.5))]),
]
augmenters=[iaa.Sometimes(0.8,[iaa.Sequential(augmenters)])]
all(os.path.isfile(os.path.join("../input/inaturalist-2019-fgvc6/train_val2019/",f)) for f in df_train.sample(n=6000).file_name)
True
tu=next(df_train.itertuples())
tu.file_name
tu.category_id
df=df_train.copy()
df["req"]=None
def imgaug_batch_gen(df,batch=16,executor=None,dmax=1024,dmin=512):
if executor is None:
executor=futures.ThreadPoolExecutor(max_workers=2)
prefetch=int(batch*2.1+1)
df_len=len(df)
def load_resize(f,dmax=dmax,dmin=dmin):
img=imageio.imread(f,pilmode="RGB")
resmin_img=np.min( img.shape[:2] )
resmin=np.clip(resmin_img,dmin,dmax)
r=resmin_img/resmin
if r>1.2 or r<1.0:
img=cv2.resize(img,None,fx=1.05/r, fy=1.05/r, interpolation = cv2.INTER_CUBIC)
return img
while True:
df=df[["category_id","file_name"]].sample(frac=1).reset_index(drop=True)
df["req"]=None
i=0
while i <(df_len-batch):
df[df.req.notnull()].loc[:i]=None
for j in range(prefetch):
try:
if df.loc[j+i,"req"] is None:
f=os.path.join("../input/inaturalist-2019-fgvc6/train_val2019/",df.loc[j+i,"file_name"])
df.loc[j+i,"req"]=executor.submit(load_resize ,f)
except KeyError:
logging.exception("imgaug_batch_gen")
df_batch=df.loc[i:i+batch-1]
imgs=[req.result() for req in df_batch.req]
resmin=np.min(np.array([im.shape[:2] for im in imgs]))
resmin=np.clip(resmin,dmin,dmax)
for j in range(batch):
x,y,_=imgs[j].shape
img_resmin=min(x,y)
r=img_resmin/resmin
if r>1.2 or r<1.0:
imgs[j]=cv2.resize(imgs[j],None,fx=1.05/r, fy=1.05/r, interpolation = cv2.INTER_CUBIC)
categories=df_batch.category_id.values.astype("int32")
yield imgaug.imgaug.Batch(images=imgs,data=categories)
df.loc[i:i+batch-1,"req"]=None
i+=batch
def batch_gen(df,batch=16,augmenters=[],executor=None,dmax=1024,dmin=512,dcrop=512):
if executor is None:
executor=futures.ThreadPoolExecutor(max_workers=2)
dmin=max(dmin,dcrop)
dmax=max(dmin,dmax)
gen =imgaug_batch_gen(df,batch=batch,executor=executor,dmax=dmax,dmin=dmin)
aug=iaa.Sequential(augmenters+[iaa.CropToFixedSize(dmin,dmin),iaa.PadToFixedSize(dmin,dmin)])
#aug_pool=aug.pool(processes=-1,maxtasksperchild=8)
#gen=aug_pool.imap_batches_unordered(gen, chunksize=1)
def aug_closure_gen(gen=gen):
b=next(gen)
fut=executor.submit(aug.augment_batches,[b],background=False)
for b in gen:
imgs=list(fut.result())[0].images_aug
fut=executor.submit(aug.augment_batches,[b],background=False)
#b=list(aug.augment_batches([b],background=False))[0]
imgs=[im[None,...] for im in imgs]
X=np.concatenate(imgs).astype("float32")/256
Y=b.data[...,None]
yield X,Y
aug_closure=aug_closure_gen(gen)
return aug_closure
aug=iaa.Sequential(augmenters)
#aug_pool=aug.pool(processes=None, maxtasksperchild=None)
gen=batch_gen(df_train,batch=16,augmenters=augmenters)
X,Y=next(gen)
del gen
b_len=X.shape[0]
cols=4
rows=b_len//cols
if b_len%cols!=0:
rows=rows+1
plt.figure(figsize=(25,25))
X.shape[0]
for n in range(b_len):
cat=Y[n][0]
plt.subplot(rows,cols,n+1)
plt.axis("off")
plt.title(cat)
plt.imshow(X[n])
pretrained_model=keras.applications.Xception(include_top=False, weights=None,pooling=None)
pretrained_model.load_weights("../input/xception/xception_weights_tf_dim_ordering_tf_kernels_notop.h5")
fichier_modele=f"{fichier_modele_base}_{pretrained_model.name}.h5"
fichier_modele
tensorflow : WARNING From /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version. Instructions for updating: Colocations handled automatically by placer.
'inaturalist_xception.h5'
def mean_anwer_guess(y_true, y_pred):
n=K.get_variable_shape(y_pred)[1]
y_pred=K.cast(y_pred,"int64")
preds=K.gather(y_true,y_pred)
return K.mean(preds)
def mean_prandom_ratio(y_true, y_pred):
n=K.get_variable_shape(y_pred)[1]
y_pred=K.cast(y_pred,"int64")
preds=K.gather(y_true,y_pred)
return K.mean(preds*n)
import functools
def get_sparse_topn__categorical_accuracy(k):
func=functools.partial(keras.metrics.sparse_top_k_categorical_accuracy,k=k)
func.__name__=f"sparse_top_{k}_categorical_accuracy"
return func
out_regulariser=keras.regularizers.l1_l2(l1=0.01, l2=0.05)
image_input=keras.Input(shape=(None,None,3), name="image_input", dtype="float32")
bottleneck1=pretrained_model(image_input)
bottleneck=keras.layers.Conv2D(filters=600,
kernel_size=3,padding="same",
kernel_initializer=keras.initializers.Orthogonal(),
activation="selu",
activity_regularizer=keras.regularizers.l1_l2(l1=0.01, l2=0.05),
strides=2
)(bottleneck1)
pool1=(keras.layers.GlobalMaxPool2D()(bottleneck1))
pool=(keras.layers.GlobalMaxPool2D()(bottleneck))
pool=keras.layers.Concatenate()([pool,pool1])
pool=keras.layers.AlphaDropout(0.3)(pool)
pre_out=keras.layers.Dense(1200 ,
name="pre_out",
activation="selu",
kernel_regularizer=out_regulariser
)(pool)
out=keras.layers.Dense(classes,
activation="softmax"
,name="out",
kernel_initializer=keras.initializers.Orthogonal(),
kernel_regularizer=out_regulariser)(pre_out)
model=keras.Model(inputs=image_input,outputs=out)
pretrained_model.trainable=False
optimizer=keras.optimizers.Adam(clipnorm=5. , clipvalue=5.,amsgrad=False,lr=0.0005)
model.compile(optimizer,
loss="sparse_categorical_crossentropy",
metrics=["sparse_categorical_accuracy","sparse_categorical_crossentropy",mean_anwer_guess,get_sparse_topn__categorical_accuracy(2),
get_sparse_topn__categorical_accuracy(5),get_sparse_topn__categorical_accuracy(10)]
)
model.summary()
__________________________________________________________________________________________________ Layer (type) Output Shape Param # Connected to ================================================================================================== image_input (InputLayer) (None, None, None, 3 0 __________________________________________________________________________________________________ xception (Model) (None, None, None, 2 20861480 image_input[0][0] __________________________________________________________________________________________________ conv2d_5 (Conv2D) (None, None, None, 6 11059800 xception[1][0] __________________________________________________________________________________________________ global_max_pooling2d_2 (GlobalM (None, 600) 0 conv2d_5[0][0] __________________________________________________________________________________________________ global_max_pooling2d_1 (GlobalM (None, 2048) 0 xception[1][0] __________________________________________________________________________________________________ concatenate_1 (Concatenate) (None, 2648) 0 global_max_pooling2d_2[0][0] global_max_pooling2d_1[0][0] __________________________________________________________________________________________________ alpha_dropout_1 (AlphaDropout) (None, 2648) 0 concatenate_1[0][0] __________________________________________________________________________________________________ pre_out (Dense) (None, 1200) 3178800 alpha_dropout_1[0][0] __________________________________________________________________________________________________ out (Dense) (None, 1010) 1213010 pre_out[0][0] ================================================================================================== Total params: 36,313,090 Trainable params: 15,451,610 Non-trainable params: 20,861,480 __________________________________________________________________________________________________
class termination_date(keras.callbacks.Callback ):
def __init__(self,end_date):
self.end_date=end_date
def on_epoch_end(self, batch, logs=None):
if datetime.datetime.now()>self.end_date:
self.model.stop_training = True
logging.info("end date")
class logcallback(keras.callbacks.Callback):
def __init__(self,logger=None):
if logger is None:
logger=logging.getLogger('traincallback')
self.logger=logger
def on_train_begin(self, logs={}):
self.logger.info("training start: %s",self.model.name)
def on_batch_end(self, batch, logs={}):
met=""
for k,v in logs.items():
met+=f"{k}: {str(v)} "
self.logger.debug("batch: %s - %s",batch,met)
def on_epoch_end(self, epoch, logs=None):
met=""
for k,v in logs.items():
met+=f"{k}: {str(v)} "
self.logger.info("epoch: %s - %s",epoch,met)
def on_train_end(self, logs={}):
self.logger.info("training end: %s",self.model.name)
callbacks=[
keras.callbacks.ReduceLROnPlateau(monitor='val_sparse_categorical_accuracy',
patience=10,
min_delta=0.0005,
factor=0.6,
#min_lr=1e-6,
verbose=1,
cooldown=5
),
keras.callbacks.ModelCheckpoint(monitor='val_sparse_categorical_accuracy',
filepath=fichier_modele,
verbose=1,
save_best_only=True,
period=20),
keras.callbacks.EarlyStopping(monitor='val_sparse_categorical_accuracy',
patience=20,
verbose=1,
restore_best_weights=True
),
keras.callbacks.CSVLogger("train.csv", separator=',', append=True),
termination_date(date_limite-datetime.timedelta(minutes=30)),
keras.callbacks.BaseLogger(),
logcallback()
]
if load_keras_weights:
for fp in glob.glob(f"../input/**/{fichier_modele}",recursive=True):
try:
model.load_weights(fp, by_name=True, skip_mismatch=True)
logging.info("loaded weights:",fb)
except Exception as e:
print(type(e),e)
logging.exception("exception loading: %s %s",fp,e)
if os.path.exists(fichier_modele):
model.load_weights(fichier_modele, by_name=True, skip_mismatch=True)
logging.info("loaded weights:",fb)
val_gen=batch_gen(df_val,batch=val_batch_size,dmax=dcrop,dmin=dcrop,dcrop=dcrop)
uptime=datetime.datetime.now()-date_depart
logging.info("pre train start %s",uptime)
root : INFO pre train start 0:00:41.700056
hist_pre=model.fit_generator(batch_gen(df_train,batch=train_batch_size,augmenters=augmenters,dmax=dmax,dmin=dmin,dcrop=dcrop),
steps_per_epoch=1280/train_batch_size,
epochs=150,
verbose=1,
validation_data=val_gen,
validation_steps=300/val_batch_size,
callbacks= callbacks+ [ keras.callbacks.EarlyStopping(monitor='val_sparse_categorical_accuracy',
patience=20,
verbose=1,
restore_best_weights=True
)]
)
model.save(fichier_modele)
logging.info("pre train end %s",datetime.datetime.now()-date_depart)
logging.info("remaining time %s",datetime.timedelta(hours=9)+date_depart-datetime.datetime.now())
tensorflow : WARNING From /opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version. Instructions for updating: Use tf.cast instead. tensorflow : WARNING From /opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version. Instructions for updating: Deprecated in favor of operator or tf.math.divide. traincallback: INFO training start: model_1
Epoch 1/150 20/20 [==============================] - 336s 17s/step - loss: 72279.2942 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 7.8264 - mean_anwer_guess: 451.1500 - sparse_top_2_categorical_accuracy: 0.0031 - sparse_top_5_categorical_accuracy: 0.0070 - sparse_top_10_categorical_accuracy: 0.0109 - val_loss: 1892.6826 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 7.3714 - val_mean_anwer_guess: 473.1333 - val_sparse_top_2_categorical_accuracy: 0.0067 - val_sparse_top_5_categorical_accuracy: 0.0100 - val_sparse_top_10_categorical_accuracy: 0.0100
traincallback: INFO epoch: 0 - val_loss: 1892.6826123046876 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 7.371352685292562 val_mean_anwer_guess: 473.1333333333333 val_sparse_top_2_categorical_accuracy: 0.006666666666666667 val_sparse_top_5_categorical_accuracy: 0.01 val_sparse_top_10_categorical_accuracy: 0.01 loss: 72279.29423828125 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 7.826378536224365 mean_anwer_guess: 451.15 sparse_top_2_categorical_accuracy: 0.003125 sparse_top_5_categorical_accuracy: 0.00703125 sparse_top_10_categorical_accuracy: 0.0109375 lr: 0.0005
Epoch 2/150 20/20 [==============================] - 331s 17s/step - loss: 22305.4429 - sparse_categorical_accuracy: 0.0031 - sparse_categorical_crossentropy: 7.3037 - mean_anwer_guess: 519.3000 - sparse_top_2_categorical_accuracy: 0.0047 - sparse_top_5_categorical_accuracy: 0.0055 - sparse_top_10_categorical_accuracy: 0.0133 - val_loss: 1158.1500 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 7.1955 - val_mean_anwer_guess: 496.5200 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0033
traincallback: INFO epoch: 1 - val_loss: 1158.1499674479167 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 7.195486653645833 val_mean_anwer_guess: 496.52 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.0033333333333333335 loss: 22305.44287109375 sparse_categorical_accuracy: 0.003125 sparse_categorical_crossentropy: 7.303728413581848 mean_anwer_guess: 519.3 sparse_top_2_categorical_accuracy: 0.0046875 sparse_top_5_categorical_accuracy: 0.00546875 sparse_top_10_categorical_accuracy: 0.01328125 lr: 0.0005
Epoch 3/150 20/20 [==============================] - 327s 16s/step - loss: 14746.6170 - sparse_categorical_accuracy: 0.0023 - sparse_categorical_crossentropy: 6.9962 - mean_anwer_guess: 366.9000 - sparse_top_2_categorical_accuracy: 0.0023 - sparse_top_5_categorical_accuracy: 0.0055 - sparse_top_10_categorical_accuracy: 0.0117 - val_loss: 713.7605 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 7.0041 - val_mean_anwer_guess: 535.3333 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0000e+00 - val_sparse_top_10_categorical_accuracy: 0.0067
traincallback: INFO epoch: 2 - val_loss: 713.7604996744792 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 7.004112447102864 val_mean_anwer_guess: 535.3333333333334 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0 val_sparse_top_10_categorical_accuracy: 0.006666666666666667 loss: 14746.6169921875 sparse_categorical_accuracy: 0.00234375 sparse_categorical_crossentropy: 6.996247887611389 mean_anwer_guess: 366.9 sparse_top_2_categorical_accuracy: 0.00234375 sparse_top_5_categorical_accuracy: 0.00546875 sparse_top_10_categorical_accuracy: 0.01171875 lr: 0.0005
Epoch 4/150 20/20 [==============================] - 332s 17s/step - loss: 11757.3366 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 6.8674 - mean_anwer_guess: 461.7000 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0117 - sparse_top_10_categorical_accuracy: 0.0227 - val_loss: 477.2980 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9355 - val_mean_anwer_guess: 510.4133 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0133
traincallback: INFO epoch: 3 - val_loss: 477.29796549479164 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.935500621795654 val_mean_anwer_guess: 510.41333333333336 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.013333333333333334 loss: 11757.33662109375 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 6.867412948608399 mean_anwer_guess: 461.7 sparse_top_2_categorical_accuracy: 0.00390625 sparse_top_5_categorical_accuracy: 0.01171875 sparse_top_10_categorical_accuracy: 0.02265625 lr: 0.0005
Epoch 5/150 20/20 [==============================] - 313s 16s/step - loss: 9759.2302 - sparse_categorical_accuracy: 0.0031 - sparse_categorical_crossentropy: 6.8926 - mean_anwer_guess: 527.5500 - sparse_top_2_categorical_accuracy: 0.0047 - sparse_top_5_categorical_accuracy: 0.0117 - sparse_top_10_categorical_accuracy: 0.0219 - val_loss: 349.2577 - val_sparse_categorical_accuracy: 0.0033 - val_sparse_categorical_crossentropy: 6.9212 - val_mean_anwer_guess: 458.2133 - val_sparse_top_2_categorical_accuracy: 0.0033 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0167
traincallback: INFO epoch: 4 - val_loss: 349.25768595377605 val_sparse_categorical_accuracy: 0.0033333333333333335 val_sparse_categorical_crossentropy: 6.921210231781006 val_mean_anwer_guess: 458.2133333333333 val_sparse_top_2_categorical_accuracy: 0.0033333333333333335 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.016666666666666666 loss: 9759.230224609375 sparse_categorical_accuracy: 0.003125 sparse_categorical_crossentropy: 6.892594838142395 mean_anwer_guess: 527.55 sparse_top_2_categorical_accuracy: 0.0046875 sparse_top_5_categorical_accuracy: 0.01171875 sparse_top_10_categorical_accuracy: 0.021875 lr: 0.0005
Epoch 6/150 20/20 [==============================] - 314s 16s/step - loss: 8383.8431 - sparse_categorical_accuracy: 0.0039 - sparse_categorical_crossentropy: 6.9110 - mean_anwer_guess: 237.0000 - sparse_top_2_categorical_accuracy: 0.0055 - sparse_top_5_categorical_accuracy: 0.0141 - sparse_top_10_categorical_accuracy: 0.0266 - val_loss: 276.3368 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9180 - val_mean_anwer_guess: 493.2267 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0100
traincallback: INFO epoch: 5 - val_loss: 276.3368009440104 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.917951691945394 val_mean_anwer_guess: 493.2266666666667 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.01 loss: 8383.843115234375 sparse_categorical_accuracy: 0.00390625 sparse_categorical_crossentropy: 6.910976839065552 mean_anwer_guess: 237.0 sparse_top_2_categorical_accuracy: 0.00546875 sparse_top_5_categorical_accuracy: 0.0140625 sparse_top_10_categorical_accuracy: 0.0265625 lr: 0.0005
Epoch 7/150 20/20 [==============================] - 316s 16s/step - loss: 7551.0763 - sparse_categorical_accuracy: 0.0031 - sparse_categorical_crossentropy: 6.9137 - mean_anwer_guess: 513.4500 - sparse_top_2_categorical_accuracy: 0.0047 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0203 - val_loss: 233.6073 - val_sparse_categorical_accuracy: 0.0033 - val_sparse_categorical_crossentropy: 6.9181 - val_mean_anwer_guess: 481.6667 - val_sparse_top_2_categorical_accuracy: 0.0033 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0133
traincallback: INFO epoch: 6 - val_loss: 233.60728352864584 val_sparse_categorical_accuracy: 0.0033333333333333335 val_sparse_categorical_crossentropy: 6.918062299092611 val_mean_anwer_guess: 481.6666666666667 val_sparse_top_2_categorical_accuracy: 0.0033333333333333335 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.013333333333333334 loss: 7551.076342773437 sparse_categorical_accuracy: 0.003125 sparse_categorical_crossentropy: 6.913731050491333 mean_anwer_guess: 513.45 sparse_top_2_categorical_accuracy: 0.0046875 sparse_top_5_categorical_accuracy: 0.00859375 sparse_top_10_categorical_accuracy: 0.0203125 lr: 0.0005
Epoch 8/150 20/20 [==============================] - 320s 16s/step - loss: 7037.9725 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 6.9135 - mean_anwer_guess: 433.8500 - sparse_top_2_categorical_accuracy: 0.0016 - sparse_top_5_categorical_accuracy: 0.0055 - sparse_top_10_categorical_accuracy: 0.0133 - val_loss: 250.0103 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9175 - val_mean_anwer_guess: 504.5200 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0000e+00 - val_sparse_top_10_categorical_accuracy: 0.0067
traincallback: INFO epoch: 7 - val_loss: 250.01030049641926 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.917485218048096 val_mean_anwer_guess: 504.52 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0 val_sparse_top_10_categorical_accuracy: 0.006666666666666667 loss: 7037.9724609375 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 6.913473224639892 mean_anwer_guess: 433.85 sparse_top_2_categorical_accuracy: 0.0015625 sparse_top_5_categorical_accuracy: 0.00546875 sparse_top_10_categorical_accuracy: 0.01328125 lr: 0.0005
Epoch 9/150 20/20 [==============================] - 321s 16s/step - loss: 6745.8842 - sparse_categorical_accuracy: 0.0000e+00 - sparse_categorical_crossentropy: 6.9122 - mean_anwer_guess: 461.6500 - sparse_top_2_categorical_accuracy: 0.0031 - sparse_top_5_categorical_accuracy: 0.0156 - sparse_top_10_categorical_accuracy: 0.0266 - val_loss: 207.7814 - val_sparse_categorical_accuracy: 0.0033 - val_sparse_categorical_crossentropy: 6.9172 - val_mean_anwer_guess: 500.9467 - val_sparse_top_2_categorical_accuracy: 0.0033 - val_sparse_top_5_categorical_accuracy: 0.0067 - val_sparse_top_10_categorical_accuracy: 0.0167
traincallback: INFO epoch: 8 - val_loss: 207.78143046061197 val_sparse_categorical_accuracy: 0.0033333333333333335 val_sparse_categorical_crossentropy: 6.917235730489095 val_mean_anwer_guess: 500.94666666666666 val_sparse_top_2_categorical_accuracy: 0.0033333333333333335 val_sparse_top_5_categorical_accuracy: 0.006666666666666667 val_sparse_top_10_categorical_accuracy: 0.016666666666666666 loss: 6745.8841796875 sparse_categorical_accuracy: 0.0 sparse_categorical_crossentropy: 6.912249827384949 mean_anwer_guess: 461.65 sparse_top_2_categorical_accuracy: 0.003125 sparse_top_5_categorical_accuracy: 0.015625 sparse_top_10_categorical_accuracy: 0.0265625 lr: 0.0005
Epoch 10/150 20/20 [==============================] - 322s 16s/step - loss: 6187.9865 - sparse_categorical_accuracy: 0.0031 - sparse_categorical_crossentropy: 6.9120 - mean_anwer_guess: 459.1500 - sparse_top_2_categorical_accuracy: 0.0047 - sparse_top_5_categorical_accuracy: 0.0102 - sparse_top_10_categorical_accuracy: 0.0195 - val_loss: 201.7798 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9166 - val_mean_anwer_guess: 451.2800 - val_sparse_top_2_categorical_accuracy: 0.0033 - val_sparse_top_5_categorical_accuracy: 0.0067 - val_sparse_top_10_categorical_accuracy: 0.0067
traincallback: INFO epoch: 9 - val_loss: 201.77976094563803 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.916606928507487 val_mean_anwer_guess: 451.28 val_sparse_top_2_categorical_accuracy: 0.0033333333333333335 val_sparse_top_5_categorical_accuracy: 0.006666666666666667 val_sparse_top_10_categorical_accuracy: 0.006666666666666667 loss: 6187.986499023437 sparse_categorical_accuracy: 0.003125 sparse_categorical_crossentropy: 6.911971855163574 mean_anwer_guess: 459.15 sparse_top_2_categorical_accuracy: 0.0046875 sparse_top_5_categorical_accuracy: 0.01015625 sparse_top_10_categorical_accuracy: 0.01953125 lr: 0.0005
Epoch 11/150
root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3030] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3030] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3031] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3031] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3032] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3032] is not in the [index]'
20/20 [==============================] - 312s 16s/step - loss: 6009.7854 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.9112 - mean_anwer_guess: 367.0500 - sparse_top_2_categorical_accuracy: 0.0031 - sparse_top_5_categorical_accuracy: 0.0094 - sparse_top_10_categorical_accuracy: 0.0187 - val_loss: 179.8454 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9179 - val_mean_anwer_guess: 437.6667 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0067
traincallback: INFO epoch: 10 - val_loss: 179.8454327392578 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.917929712931315 val_mean_anwer_guess: 437.6666666666667 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.006666666666666667 loss: 6009.785375976562 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.9111556053161625 mean_anwer_guess: 367.05 sparse_top_2_categorical_accuracy: 0.003125 sparse_top_5_categorical_accuracy: 0.009375 sparse_top_10_categorical_accuracy: 0.01875 lr: 0.0005
Epoch 12/150 20/20 [==============================] - 321s 16s/step - loss: 5702.8190 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.9107 - mean_anwer_guess: 449.5000 - sparse_top_2_categorical_accuracy: 0.0016 - sparse_top_5_categorical_accuracy: 0.0055 - sparse_top_10_categorical_accuracy: 0.0172 - val_loss: 180.1914 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9188 - val_mean_anwer_guess: 521.4000 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0000e+00 - val_sparse_top_10_categorical_accuracy: 0.0100
traincallback: INFO epoch: 11 - val_loss: 180.19140665690105 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.918752930959066 val_mean_anwer_guess: 521.4 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0 val_sparse_top_10_categorical_accuracy: 0.01 loss: 5702.8189697265625 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.910686731338501 mean_anwer_guess: 449.5 sparse_top_2_categorical_accuracy: 0.0015625 sparse_top_5_categorical_accuracy: 0.00546875 sparse_top_10_categorical_accuracy: 0.0171875 lr: 0.0005
Epoch 13/150 20/20 [==============================] - 334s 17s/step - loss: 5418.8233 - sparse_categorical_accuracy: 0.0000e+00 - sparse_categorical_crossentropy: 6.9098 - mean_anwer_guess: 411.1500 - sparse_top_2_categorical_accuracy: 7.8125e-04 - sparse_top_5_categorical_accuracy: 0.0109 - sparse_top_10_categorical_accuracy: 0.0211 - val_loss: 168.6256 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9169 - val_mean_anwer_guess: 554.9200 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0067 - val_sparse_top_10_categorical_accuracy: 0.0067
traincallback: INFO epoch: 12 - val_loss: 168.62560038248697 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.916946201324463 val_mean_anwer_guess: 554.92 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.006666666666666667 val_sparse_top_10_categorical_accuracy: 0.006666666666666667 loss: 5418.823291015625 sparse_categorical_accuracy: 0.0 sparse_categorical_crossentropy: 6.909760808944702 mean_anwer_guess: 411.15 sparse_top_2_categorical_accuracy: 0.00078125 sparse_top_5_categorical_accuracy: 0.0109375 sparse_top_10_categorical_accuracy: 0.02109375 lr: 0.0005
Epoch 14/150 20/20 [==============================] - 331s 17s/step - loss: 5436.2248 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.9085 - mean_anwer_guess: 517.9500 - sparse_top_2_categorical_accuracy: 0.0023 - sparse_top_5_categorical_accuracy: 0.0094 - sparse_top_10_categorical_accuracy: 0.0148 - val_loss: 186.9243 - val_sparse_categorical_accuracy: 0.0067 - val_sparse_categorical_crossentropy: 6.9170 - val_mean_anwer_guess: 492.3333 - val_sparse_top_2_categorical_accuracy: 0.0067 - val_sparse_top_5_categorical_accuracy: 0.0067 - val_sparse_top_10_categorical_accuracy: 0.0100
traincallback: INFO epoch: 13 - val_loss: 186.92434041341147 val_sparse_categorical_accuracy: 0.006666666666666667 val_sparse_categorical_crossentropy: 6.916962973276774 val_mean_anwer_guess: 492.3333333333333 val_sparse_top_2_categorical_accuracy: 0.006666666666666667 val_sparse_top_5_categorical_accuracy: 0.006666666666666667 val_sparse_top_10_categorical_accuracy: 0.01 loss: 5436.224755859375 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.908514475822448 mean_anwer_guess: 517.95 sparse_top_2_categorical_accuracy: 0.00234375 sparse_top_5_categorical_accuracy: 0.009375 sparse_top_10_categorical_accuracy: 0.01484375 lr: 0.0005
Epoch 15/150 20/20 [==============================] - 323s 16s/step - loss: 5270.2783 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 6.9088 - mean_anwer_guess: 505.9500 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0187 - val_loss: 180.5704 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9181 - val_mean_anwer_guess: 524.0800 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0000e+00 - val_sparse_top_10_categorical_accuracy: 0.0033
traincallback: INFO epoch: 14 - val_loss: 180.57043375651043 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.918078174591065 val_mean_anwer_guess: 524.08 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0 val_sparse_top_10_categorical_accuracy: 0.0033333333333333335 loss: 5270.278271484375 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 6.90878643989563 mean_anwer_guess: 505.95 sparse_top_2_categorical_accuracy: 0.00390625 sparse_top_5_categorical_accuracy: 0.00859375 sparse_top_10_categorical_accuracy: 0.01875 lr: 0.0005
Epoch 16/150 20/20 [==============================] - 325s 16s/step - loss: 5393.2199 - sparse_categorical_accuracy: 0.0023 - sparse_categorical_crossentropy: 6.9075 - mean_anwer_guess: 470.8000 - sparse_top_2_categorical_accuracy: 0.0023 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0180 - val_loss: 179.1251 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9160 - val_mean_anwer_guess: 521.4133 - val_sparse_top_2_categorical_accuracy: 0.0033 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0067
traincallback: INFO epoch: 15 - val_loss: 179.12506388346355 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.9159996096293135 val_mean_anwer_guess: 521.4133333333333 val_sparse_top_2_categorical_accuracy: 0.0033333333333333335 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.006666666666666667 loss: 5393.219946289062 sparse_categorical_accuracy: 0.00234375 sparse_categorical_crossentropy: 6.90752956867218 mean_anwer_guess: 470.8 sparse_top_2_categorical_accuracy: 0.00234375 sparse_top_5_categorical_accuracy: 0.00859375 sparse_top_10_categorical_accuracy: 0.01796875 lr: 0.0005
Epoch 17/150 20/20 [==============================] - 318s 16s/step - loss: 4978.4317 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.9074 - mean_anwer_guess: 443.1500 - sparse_top_2_categorical_accuracy: 0.0031 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0203 - val_loss: 221.3766 - val_sparse_categorical_accuracy: 0.0033 - val_sparse_categorical_crossentropy: 6.9185 - val_mean_anwer_guess: 545.5333 - val_sparse_top_2_categorical_accuracy: 0.0067 - val_sparse_top_5_categorical_accuracy: 0.0067 - val_sparse_top_10_categorical_accuracy: 0.0133
traincallback: INFO epoch: 16 - val_loss: 221.37662109375 val_sparse_categorical_accuracy: 0.0033333333333333335 val_sparse_categorical_crossentropy: 6.918507448832194 val_mean_anwer_guess: 545.5333333333333 val_sparse_top_2_categorical_accuracy: 0.006666666666666667 val_sparse_top_5_categorical_accuracy: 0.006666666666666667 val_sparse_top_10_categorical_accuracy: 0.013333333333333334 loss: 4978.431665039063 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.9073686599731445 mean_anwer_guess: 443.15 sparse_top_2_categorical_accuracy: 0.003125 sparse_top_5_categorical_accuracy: 0.00859375 sparse_top_10_categorical_accuracy: 0.0203125 lr: 0.0005
Epoch 18/150 20/20 [==============================] - 327s 16s/step - loss: 5361.6376 - sparse_categorical_accuracy: 0.0023 - sparse_categorical_crossentropy: 6.9069 - mean_anwer_guess: 588.4000 - sparse_top_2_categorical_accuracy: 0.0031 - sparse_top_5_categorical_accuracy: 0.0094 - sparse_top_10_categorical_accuracy: 0.0172 - val_loss: 164.1036 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9183 - val_mean_anwer_guess: 473.7467 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0133
traincallback: INFO epoch: 17 - val_loss: 164.10355163574218 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.918310362497966 val_mean_anwer_guess: 473.74666666666667 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.013333333333333334 loss: 5361.637622070312 sparse_categorical_accuracy: 0.00234375 sparse_categorical_crossentropy: 6.906924247741699 mean_anwer_guess: 588.4 sparse_top_2_categorical_accuracy: 0.003125 sparse_top_5_categorical_accuracy: 0.009375 sparse_top_10_categorical_accuracy: 0.0171875 lr: 0.0005
Epoch 19/150 20/20 [==============================] - 323s 16s/step - loss: 4651.0018 - sparse_categorical_accuracy: 0.0047 - sparse_categorical_crossentropy: 6.9048 - mean_anwer_guess: 447.4500 - sparse_top_2_categorical_accuracy: 0.0055 - sparse_top_5_categorical_accuracy: 0.0094 - sparse_top_10_categorical_accuracy: 0.0203 - val_loss: 146.1939 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9199 - val_mean_anwer_guess: 490.9600 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0100
traincallback: INFO epoch: 18 - val_loss: 146.19388153076173 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.919873574574789 val_mean_anwer_guess: 490.96 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.01 loss: 4651.0018310546875 sparse_categorical_accuracy: 0.0046875 sparse_categorical_crossentropy: 6.904846167564392 mean_anwer_guess: 447.45 sparse_top_2_categorical_accuracy: 0.00546875 sparse_top_5_categorical_accuracy: 0.009375 sparse_top_10_categorical_accuracy: 0.0203125 lr: 0.0005
Epoch 20/150 20/20 [==============================] - 325s 16s/step - loss: 5107.9682 - sparse_categorical_accuracy: 0.0023 - sparse_categorical_crossentropy: 6.9058 - mean_anwer_guess: 463.1500 - sparse_top_2_categorical_accuracy: 0.0031 - sparse_top_5_categorical_accuracy: 0.0047 - sparse_top_10_categorical_accuracy: 0.0133 - val_loss: 153.7441 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9174 - val_mean_anwer_guess: 514.1467 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0067 Epoch 00020: val_sparse_categorical_accuracy improved from -inf to 0.00000, saving model to inaturalist_xception.h5
traincallback: INFO epoch: 19 - val_loss: 153.74413940429687 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.917436389923096 val_mean_anwer_guess: 514.1466666666666 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.006666666666666667 loss: 5107.968176269531 sparse_categorical_accuracy: 0.00234375 sparse_categorical_crossentropy: 6.905842852592468 mean_anwer_guess: 463.15 sparse_top_2_categorical_accuracy: 0.003125 sparse_top_5_categorical_accuracy: 0.0046875 sparse_top_10_categorical_accuracy: 0.01328125 lr: 0.0005
Epoch 21/150 19/20 [===========================>..] - ETA: 12s - loss: 4671.2478 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.9031 - mean_anwer_guess: 528.1053 - sparse_top_2_categorical_accuracy: 0.0041 - sparse_top_5_categorical_accuracy: 0.0082 - sparse_top_10_categorical_accuracy: 0.0132
root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3030] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3030] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3031] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3031] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3032] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3032] is not in the [index]'
20/20 [==============================] - 323s 16s/step - loss: 4658.8865 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.9031 - mean_anwer_guess: 510.5000 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0078 - sparse_top_10_categorical_accuracy: 0.0125 - val_loss: 140.8388 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9200 - val_mean_anwer_guess: 471.5333 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0000e+00 - val_sparse_top_10_categorical_accuracy: 0.0000e+00
traincallback: INFO epoch: 20 - val_loss: 140.83882486979167 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.919992713928223 val_mean_anwer_guess: 471.53333333333336 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0 val_sparse_top_10_categorical_accuracy: 0.0 loss: 4658.886547851563 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.903148984909057 mean_anwer_guess: 510.5 sparse_top_2_categorical_accuracy: 0.00390625 sparse_top_5_categorical_accuracy: 0.0078125 sparse_top_10_categorical_accuracy: 0.0125 lr: 0.0005
Epoch 22/150 20/20 [==============================] - 333s 17s/step - loss: 4933.3177 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.9026 - mean_anwer_guess: 446.8000 - sparse_top_2_categorical_accuracy: 0.0031 - sparse_top_5_categorical_accuracy: 0.0055 - sparse_top_10_categorical_accuracy: 0.0141 - val_loss: 170.8401 - val_sparse_categorical_accuracy: 0.0033 - val_sparse_categorical_crossentropy: 6.9207 - val_mean_anwer_guess: 514.9200 - val_sparse_top_2_categorical_accuracy: 0.0067 - val_sparse_top_5_categorical_accuracy: 0.0133 - val_sparse_top_10_categorical_accuracy: 0.0200
traincallback: INFO epoch: 21 - val_loss: 170.8401141357422 val_sparse_categorical_accuracy: 0.0033333333333333335 val_sparse_categorical_crossentropy: 6.920669473012288 val_mean_anwer_guess: 514.92 val_sparse_top_2_categorical_accuracy: 0.006666666666666667 val_sparse_top_5_categorical_accuracy: 0.013333333333333334 val_sparse_top_10_categorical_accuracy: 0.02 loss: 4933.317687988281 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.9026219844818115 mean_anwer_guess: 446.8 sparse_top_2_categorical_accuracy: 0.003125 sparse_top_5_categorical_accuracy: 0.00546875 sparse_top_10_categorical_accuracy: 0.0140625 lr: 0.0005
Epoch 23/150 20/20 [==============================] - 338s 17s/step - loss: 4780.6695 - sparse_categorical_accuracy: 0.0031 - sparse_categorical_crossentropy: 6.9022 - mean_anwer_guess: 466.6000 - sparse_top_2_categorical_accuracy: 0.0047 - sparse_top_5_categorical_accuracy: 0.0094 - sparse_top_10_categorical_accuracy: 0.0180 - val_loss: 190.9515 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9179 - val_mean_anwer_guess: 502.5733 - val_sparse_top_2_categorical_accuracy: 0.0033 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0133
traincallback: INFO epoch: 22 - val_loss: 190.95153991699217 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.9179022471110025 val_mean_anwer_guess: 502.5733333333333 val_sparse_top_2_categorical_accuracy: 0.0033333333333333335 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.013333333333333334 loss: 4780.669470214843 sparse_categorical_accuracy: 0.003125 sparse_categorical_crossentropy: 6.902244973182678 mean_anwer_guess: 466.6 sparse_top_2_categorical_accuracy: 0.0046875 sparse_top_5_categorical_accuracy: 0.009375 sparse_top_10_categorical_accuracy: 0.01796875 lr: 0.0005
Epoch 24/150 20/20 [==============================] - 331s 17s/step - loss: 4796.8529 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.9015 - mean_anwer_guess: 393.7000 - sparse_top_2_categorical_accuracy: 0.0016 - sparse_top_5_categorical_accuracy: 0.0102 - sparse_top_10_categorical_accuracy: 0.0180 - val_loss: 179.2027 - val_sparse_categorical_accuracy: 0.0033 - val_sparse_categorical_crossentropy: 6.9161 - val_mean_anwer_guess: 473.8400 - val_sparse_top_2_categorical_accuracy: 0.0033 - val_sparse_top_5_categorical_accuracy: 0.0067 - val_sparse_top_10_categorical_accuracy: 0.0133
traincallback: INFO epoch: 23 - val_loss: 179.20270853678386 val_sparse_categorical_accuracy: 0.0033333333333333335 val_sparse_categorical_crossentropy: 6.916128889719645 val_mean_anwer_guess: 473.84 val_sparse_top_2_categorical_accuracy: 0.0033333333333333335 val_sparse_top_5_categorical_accuracy: 0.006666666666666667 val_sparse_top_10_categorical_accuracy: 0.013333333333333334 loss: 4796.852868652344 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.9015449523925785 mean_anwer_guess: 393.7 sparse_top_2_categorical_accuracy: 0.0015625 sparse_top_5_categorical_accuracy: 0.01015625 sparse_top_10_categorical_accuracy: 0.01796875 lr: 0.0005
Epoch 00024: ReduceLROnPlateau reducing learning rate to 0.0003000000142492354. Epoch 25/150 20/20 [==============================] - 327s 16s/step - loss: 3995.6191 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 6.9012 - mean_anwer_guess: 497.9000 - sparse_top_2_categorical_accuracy: 0.0016 - sparse_top_5_categorical_accuracy: 0.0078 - sparse_top_10_categorical_accuracy: 0.0133 - val_loss: 119.3189 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9167 - val_mean_anwer_guess: 502.7200 - val_sparse_top_2_categorical_accuracy: 0.0033 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0067
traincallback: INFO epoch: 24 - val_loss: 119.31890045166016 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.916702817281087 val_mean_anwer_guess: 502.72 val_sparse_top_2_categorical_accuracy: 0.0033333333333333335 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.006666666666666667 loss: 3995.6191162109376 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 6.901170468330383 mean_anwer_guess: 497.9 sparse_top_2_categorical_accuracy: 0.0015625 sparse_top_5_categorical_accuracy: 0.0078125 sparse_top_10_categorical_accuracy: 0.01328125 lr: 0.0003
Epoch 26/150 20/20 [==============================] - 328s 16s/step - loss: 3407.6282 - sparse_categorical_accuracy: 0.0000e+00 - sparse_categorical_crossentropy: 6.9009 - mean_anwer_guess: 325.7000 - sparse_top_2_categorical_accuracy: 0.0000e+00 - sparse_top_5_categorical_accuracy: 0.0070 - sparse_top_10_categorical_accuracy: 0.0180 - val_loss: 107.8284 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9181 - val_mean_anwer_guess: 417.0800 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0100 - val_sparse_top_10_categorical_accuracy: 0.0133
traincallback: INFO epoch: 25 - val_loss: 107.82840647379557 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.9180589866638185 val_mean_anwer_guess: 417.08 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.01 val_sparse_top_10_categorical_accuracy: 0.013333333333333334 loss: 3407.6282470703127 sparse_categorical_accuracy: 0.0 sparse_categorical_crossentropy: 6.900906300544738 mean_anwer_guess: 325.7 sparse_top_2_categorical_accuracy: 0.0 sparse_top_5_categorical_accuracy: 0.00703125 sparse_top_10_categorical_accuracy: 0.01796875 lr: 0.0003
Epoch 27/150 20/20 [==============================] - 330s 16s/step - loss: 3168.7803 - sparse_categorical_accuracy: 0.0031 - sparse_categorical_crossentropy: 6.8998 - mean_anwer_guess: 420.0000 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0078 - sparse_top_10_categorical_accuracy: 0.0172 - val_loss: 98.6710 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9190 - val_mean_anwer_guess: 526.6800 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0067
traincallback: INFO epoch: 26 - val_loss: 98.67104654947917 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.918981240590413 val_mean_anwer_guess: 526.68 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.006666666666666667 loss: 3168.7803466796877 sparse_categorical_accuracy: 0.003125 sparse_categorical_crossentropy: 6.899770498275757 mean_anwer_guess: 420.0 sparse_top_2_categorical_accuracy: 0.00390625 sparse_top_5_categorical_accuracy: 0.0078125 sparse_top_10_categorical_accuracy: 0.0171875 lr: 0.0003
Epoch 28/150 20/20 [==============================] - 342s 17s/step - loss: 3400.8443 - sparse_categorical_accuracy: 0.0031 - sparse_categorical_crossentropy: 6.8982 - mean_anwer_guess: 461.4000 - sparse_top_2_categorical_accuracy: 0.0055 - sparse_top_5_categorical_accuracy: 0.0125 - sparse_top_10_categorical_accuracy: 0.0258 - val_loss: 94.9354 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9178 - val_mean_anwer_guess: 458.9067 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0000e+00 - val_sparse_top_10_categorical_accuracy: 0.0033
traincallback: INFO epoch: 27 - val_loss: 94.93542175292968 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.917813180287679 val_mean_anwer_guess: 458.9066666666667 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0 val_sparse_top_10_categorical_accuracy: 0.0033333333333333335 loss: 3400.8443359375 sparse_categorical_accuracy: 0.003125 sparse_categorical_crossentropy: 6.898177647590638 mean_anwer_guess: 461.4 sparse_top_2_categorical_accuracy: 0.00546875 sparse_top_5_categorical_accuracy: 0.0125 sparse_top_10_categorical_accuracy: 0.02578125 lr: 0.0003
Epoch 29/150 20/20 [==============================] - 322s 16s/step - loss: 3169.0374 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.8987 - mean_anwer_guess: 577.0000 - sparse_top_2_categorical_accuracy: 0.0023 - sparse_top_5_categorical_accuracy: 0.0109 - sparse_top_10_categorical_accuracy: 0.0164 - val_loss: 95.6620 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9161 - val_mean_anwer_guess: 457.8800 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0033
traincallback: INFO epoch: 28 - val_loss: 95.66195810953776 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.916140003204346 val_mean_anwer_guess: 457.88 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.0033333333333333335 loss: 3169.0373779296874 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.898715519905091 mean_anwer_guess: 577.0 sparse_top_2_categorical_accuracy: 0.00234375 sparse_top_5_categorical_accuracy: 0.0109375 sparse_top_10_categorical_accuracy: 0.01640625 lr: 0.0003
Epoch 30/150 20/20 [==============================] - 329s 16s/step - loss: 3284.2957 - sparse_categorical_accuracy: 0.0031 - sparse_categorical_crossentropy: 6.8982 - mean_anwer_guess: 441.1500 - sparse_top_2_categorical_accuracy: 0.0063 - sparse_top_5_categorical_accuracy: 0.0141 - sparse_top_10_categorical_accuracy: 0.0242 - val_loss: 95.3214 - val_sparse_categorical_accuracy: 0.0033 - val_sparse_categorical_crossentropy: 6.9179 - val_mean_anwer_guess: 498.9200 - val_sparse_top_2_categorical_accuracy: 0.0033 - val_sparse_top_5_categorical_accuracy: 0.0067 - val_sparse_top_10_categorical_accuracy: 0.0100
traincallback: INFO epoch: 29 - val_loss: 95.32142049153646 val_sparse_categorical_accuracy: 0.0033333333333333335 val_sparse_categorical_crossentropy: 6.917855644226075 val_mean_anwer_guess: 498.92 val_sparse_top_2_categorical_accuracy: 0.0033333333333333335 val_sparse_top_5_categorical_accuracy: 0.006666666666666667 val_sparse_top_10_categorical_accuracy: 0.01 loss: 3284.295703125 sparse_categorical_accuracy: 0.003125 sparse_categorical_crossentropy: 6.8982326030731205 mean_anwer_guess: 441.15 sparse_top_2_categorical_accuracy: 0.00625 sparse_top_5_categorical_accuracy: 0.0140625 sparse_top_10_categorical_accuracy: 0.02421875 lr: 0.0003
Epoch 31/150 19/20 [===========================>..] - ETA: 12s - loss: 3104.8761 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.8976 - mean_anwer_guess: 481.2632 - sparse_top_2_categorical_accuracy: 0.0049 - sparse_top_5_categorical_accuracy: 0.0090 - sparse_top_10_categorical_accuracy: 0.0189
root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3030] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3030] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3031] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3031] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3032] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3032] is not in the [index]'
20/20 [==============================] - 316s 16s/step - loss: 3102.6513 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.8976 - mean_anwer_guess: 505.1500 - sparse_top_2_categorical_accuracy: 0.0047 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0187 - val_loss: 113.0610 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9211 - val_mean_anwer_guess: 425.9467 - val_sparse_top_2_categorical_accuracy: 0.0033 - val_sparse_top_5_categorical_accuracy: 0.0033 - val_sparse_top_10_categorical_accuracy: 0.0067
traincallback: INFO epoch: 30 - val_loss: 113.06103647867839 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.921059652964274 val_mean_anwer_guess: 425.94666666666666 val_sparse_top_2_categorical_accuracy: 0.0033333333333333335 val_sparse_top_5_categorical_accuracy: 0.0033333333333333335 val_sparse_top_10_categorical_accuracy: 0.006666666666666667 loss: 3102.65126953125 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.897599792480468 mean_anwer_guess: 505.15 sparse_top_2_categorical_accuracy: 0.0046875 sparse_top_5_categorical_accuracy: 0.00859375 sparse_top_10_categorical_accuracy: 0.01875 lr: 0.0003
Epoch 32/150 20/20 [==============================] - 322s 16s/step - loss: 3015.0873 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.8968 - mean_anwer_guess: 426.7500 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0164 - val_loss: 91.9944 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9179 - val_mean_anwer_guess: 525.2800 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0000e+00 - val_sparse_top_10_categorical_accuracy: 0.0033
traincallback: INFO epoch: 31 - val_loss: 91.99441772460938 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.91790880203247 val_mean_anwer_guess: 525.28 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0 val_sparse_top_10_categorical_accuracy: 0.0033333333333333335 loss: 3015.0873168945313 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.896802258491516 mean_anwer_guess: 426.75 sparse_top_2_categorical_accuracy: 0.00390625 sparse_top_5_categorical_accuracy: 0.00859375 sparse_top_10_categorical_accuracy: 0.01640625 lr: 0.0003
Epoch 33/150 20/20 [==============================] - 327s 16s/step - loss: 3070.9360 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 6.8980 - mean_anwer_guess: 474.6000 - sparse_top_2_categorical_accuracy: 0.0023 - sparse_top_5_categorical_accuracy: 0.0055 - sparse_top_10_categorical_accuracy: 0.0141 - val_loss: 95.6509 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9154 - val_mean_anwer_guess: 470.7600 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0000e+00 - val_sparse_top_10_categorical_accuracy: 0.0133
traincallback: INFO epoch: 32 - val_loss: 95.65090382893881 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.9153985595703125 val_mean_anwer_guess: 470.76 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0 val_sparse_top_10_categorical_accuracy: 0.013333333333333334 loss: 3070.9359619140623 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 6.897995805740356 mean_anwer_guess: 474.6 sparse_top_2_categorical_accuracy: 0.00234375 sparse_top_5_categorical_accuracy: 0.00546875 sparse_top_10_categorical_accuracy: 0.0140625 lr: 0.0003
Epoch 34/150 20/20 [==============================] - 326s 16s/step - loss: 3158.7775 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 6.8959 - mean_anwer_guess: 486.2000 - sparse_top_2_categorical_accuracy: 0.0031 - sparse_top_5_categorical_accuracy: 0.0063 - sparse_top_10_categorical_accuracy: 0.0180 - val_loss: 95.4145 - val_sparse_categorical_accuracy: 0.0033 - val_sparse_categorical_crossentropy: 6.9177 - val_mean_anwer_guess: 497.6400 - val_sparse_top_2_categorical_accuracy: 0.0033 - val_sparse_top_5_categorical_accuracy: 0.0100 - val_sparse_top_10_categorical_accuracy: 0.0133 Restoring model weights from the end of the best epoch
traincallback: INFO epoch: 33 - val_loss: 95.41447835286458 val_sparse_categorical_accuracy: 0.0033333333333333335 val_sparse_categorical_crossentropy: 6.917749404907227 val_mean_anwer_guess: 497.64 val_sparse_top_2_categorical_accuracy: 0.0033333333333333335 val_sparse_top_5_categorical_accuracy: 0.01 val_sparse_top_10_categorical_accuracy: 0.013333333333333334 loss: 3158.7774658203125 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 6.89590904712677 mean_anwer_guess: 486.2 sparse_top_2_categorical_accuracy: 0.003125 sparse_top_5_categorical_accuracy: 0.00625 sparse_top_10_categorical_accuracy: 0.01796875 lr: 0.0003
Restoring model weights from the end of the best epoch
traincallback: INFO training end: model_1
Epoch 00034: early stopping Epoch 00034: early stopping
root : INFO pre train end 3:05:34.855935 root : INFO remaining time 5:54:25.142503
pretrained_model.trainable=True
optimizer=keras.optimizers.Adam(clipnorm=5. , clipvalue=5.,amsgrad=True)
model.compile(optimizer,
loss="sparse_categorical_crossentropy",
metrics=model.metrics
)
model.summary()
__________________________________________________________________________________________________ Layer (type) Output Shape Param # Connected to ================================================================================================== image_input (InputLayer) (None, None, None, 3 0 __________________________________________________________________________________________________ xception (Model) (None, None, None, 2 20861480 image_input[0][0] __________________________________________________________________________________________________ conv2d_5 (Conv2D) (None, None, None, 6 11059800 xception[1][0] __________________________________________________________________________________________________ global_max_pooling2d_2 (GlobalM (None, 600) 0 conv2d_5[0][0] __________________________________________________________________________________________________ global_max_pooling2d_1 (GlobalM (None, 2048) 0 xception[1][0] __________________________________________________________________________________________________ concatenate_1 (Concatenate) (None, 2648) 0 global_max_pooling2d_2[0][0] global_max_pooling2d_1[0][0] __________________________________________________________________________________________________ alpha_dropout_1 (AlphaDropout) (None, 2648) 0 concatenate_1[0][0] __________________________________________________________________________________________________ pre_out (Dense) (None, 1200) 3178800 alpha_dropout_1[0][0] __________________________________________________________________________________________________ out (Dense) (None, 1010) 1213010 pre_out[0][0] ================================================================================================== Total params: 36,313,090 Trainable params: 36,258,562 Non-trainable params: 54,528 __________________________________________________________________________________________________
uptime=datetime.datetime.now()-date_depart
logging.info("main train %s",uptime)
root : INFO main train 3:05:37.701667
hist=model.fit_generator(batch_gen(df_train,batch=full_train_batch_size,augmenters=augmenters,dmax=dmax,dmin=dmin,dcrop=dcrop),
steps_per_epoch=1280/full_train_batch_size, epochs=epochs, verbose=1,
validation_data=val_gen,
validation_steps=400/val_batch_size,
callbacks= callbacks
)
model.save(fichier_modele)
print(datetime.datetime.now()-date_depart)
traincallback: INFO training start: model_1
Epoch 1/100000000 320/320 [==============================] - 827s 3s/step - loss: 48.1410 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.8948 - mean_anwer_guess: 448.5000 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0180 - val_loss: 12.1783 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9228 - val_mean_anwer_guess: 505.3300 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0000e+00 - val_sparse_top_10_categorical_accuracy: 0.0050
traincallback: INFO epoch: 0 - val_loss: 12.17833743095398 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.922817440032959 val_mean_anwer_guess: 505.33 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0 val_sparse_top_10_categorical_accuracy: 0.005 loss: 48.14103074967861 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.894803048670292 mean_anwer_guess: 448.5 sparse_top_2_categorical_accuracy: 0.00390625 sparse_top_5_categorical_accuracy: 0.00859375 sparse_top_10_categorical_accuracy: 0.01796875 lr: 0.001
Epoch 2/100000000 320/320 [==============================] - 816s 3s/step - loss: 12.4288 - sparse_categorical_accuracy: 0.0023 - sparse_categorical_crossentropy: 6.8629 - mean_anwer_guess: 441.7938 - sparse_top_2_categorical_accuracy: 0.0063 - sparse_top_5_categorical_accuracy: 0.0156 - sparse_top_10_categorical_accuracy: 0.0250 - val_loss: 12.6009 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9368 - val_mean_anwer_guess: 506.7000 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0025 - val_sparse_top_10_categorical_accuracy: 0.0075
traincallback: INFO epoch: 1 - val_loss: 12.600927228927612 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.936796236038208 val_mean_anwer_guess: 506.7 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0025 val_sparse_top_10_categorical_accuracy: 0.0075 loss: 12.428814068436623 sparse_categorical_accuracy: 0.00234375 sparse_categorical_crossentropy: 6.8629228070378305 mean_anwer_guess: 441.79375 sparse_top_2_categorical_accuracy: 0.00625 sparse_top_5_categorical_accuracy: 0.015625 sparse_top_10_categorical_accuracy: 0.025 lr: 0.001
Epoch 3/100000000 320/320 [==============================] - 819s 3s/step - loss: 12.6151 - sparse_categorical_accuracy: 0.0047 - sparse_categorical_crossentropy: 6.8430 - mean_anwer_guess: 454.4031 - sparse_top_2_categorical_accuracy: 0.0078 - sparse_top_5_categorical_accuracy: 0.0141 - sparse_top_10_categorical_accuracy: 0.0219 - val_loss: 12.6996 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9636 - val_mean_anwer_guess: 509.5800 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0025 - val_sparse_top_10_categorical_accuracy: 0.0025
traincallback: INFO epoch: 2 - val_loss: 12.69960337638855 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.9635548448562625 val_mean_anwer_guess: 509.58 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0025 val_sparse_top_10_categorical_accuracy: 0.0025 loss: 12.615091639757157 sparse_categorical_accuracy: 0.0046875 sparse_categorical_crossentropy: 6.842999868094921 mean_anwer_guess: 454.403125 sparse_top_2_categorical_accuracy: 0.0078125 sparse_top_5_categorical_accuracy: 0.0140625 sparse_top_10_categorical_accuracy: 0.021875 lr: 0.001
Epoch 4/100000000 320/320 [==============================] - 798s 2s/step - loss: 12.6081 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.8457 - mean_anwer_guess: 471.2844 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0164 - val_loss: 12.3423 - val_sparse_categorical_accuracy: 0.0025 - val_sparse_categorical_crossentropy: 6.9547 - val_mean_anwer_guess: 486.5700 - val_sparse_top_2_categorical_accuracy: 0.0075 - val_sparse_top_5_categorical_accuracy: 0.0100 - val_sparse_top_10_categorical_accuracy: 0.0125
traincallback: INFO epoch: 3 - val_loss: 12.342320804595948 val_sparse_categorical_accuracy: 0.0025 val_sparse_categorical_crossentropy: 6.954720191955566 val_mean_anwer_guess: 486.57 val_sparse_top_2_categorical_accuracy: 0.0075 val_sparse_top_5_categorical_accuracy: 0.01 val_sparse_top_10_categorical_accuracy: 0.0125 loss: 12.608069211244583 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.84574968367815 mean_anwer_guess: 471.284375 sparse_top_2_categorical_accuracy: 0.00390625 sparse_top_5_categorical_accuracy: 0.00859375 sparse_top_10_categorical_accuracy: 0.01640625 lr: 0.001
Epoch 5/100000000 319/320 [============================>.] - ETA: 2s - loss: 12.6846 - sparse_categorical_accuracy: 0.0024 - sparse_categorical_crossentropy: 6.8427 - mean_anwer_guess: 471.7962 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0165
root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3030] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3030] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3031] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3031] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3032] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3032] is not in the [index]'
320/320 [==============================] - 807s 3s/step - loss: 12.6841 - sparse_categorical_accuracy: 0.0023 - sparse_categorical_crossentropy: 6.8420 - mean_anwer_guess: 471.2313 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0164 - val_loss: 12.8512 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9653 - val_mean_anwer_guess: 520.8500 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0025 - val_sparse_top_10_categorical_accuracy: 0.0125
traincallback: INFO epoch: 4 - val_loss: 12.851205167770386 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.9653181552886965 val_mean_anwer_guess: 520.85 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0025 val_sparse_top_10_categorical_accuracy: 0.0125 loss: 12.684120106697083 sparse_categorical_accuracy: 0.00234375 sparse_categorical_crossentropy: 6.842004343867302 mean_anwer_guess: 471.23125 sparse_top_2_categorical_accuracy: 0.00390625 sparse_top_5_categorical_accuracy: 0.00859375 sparse_top_10_categorical_accuracy: 0.01640625 lr: 0.001
Epoch 6/100000000 320/320 [==============================] - 807s 3s/step - loss: 12.7279 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 6.8252 - mean_anwer_guess: 444.2563 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0063 - sparse_top_10_categorical_accuracy: 0.0141 - val_loss: 12.5491 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9697 - val_mean_anwer_guess: 464.0200 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0050 - val_sparse_top_10_categorical_accuracy: 0.0100 Epoch 00006: val_sparse_categorical_accuracy did not improve from 0.00000
traincallback: INFO epoch: 5 - val_loss: 12.549130172729493 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.969736785888672 val_mean_anwer_guess: 464.02 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.005 val_sparse_top_10_categorical_accuracy: 0.01 loss: 12.727933862805367 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 6.825203020870686 mean_anwer_guess: 444.25625 sparse_top_2_categorical_accuracy: 0.00390625 sparse_top_5_categorical_accuracy: 0.00625 sparse_top_10_categorical_accuracy: 0.0140625 lr: 0.001
Epoch 7/100000000 320/320 [==============================] - 804s 3s/step - loss: 12.7585 - sparse_categorical_accuracy: 0.0023 - sparse_categorical_crossentropy: 6.8074 - mean_anwer_guess: 429.9875 - sparse_top_2_categorical_accuracy: 0.0055 - sparse_top_5_categorical_accuracy: 0.0125 - sparse_top_10_categorical_accuracy: 0.0211 - val_loss: 12.7553 - val_sparse_categorical_accuracy: 0.0025 - val_sparse_categorical_crossentropy: 6.9717 - val_mean_anwer_guess: 526.0300 - val_sparse_top_2_categorical_accuracy: 0.0050 - val_sparse_top_5_categorical_accuracy: 0.0100 - val_sparse_top_10_categorical_accuracy: 0.0225
traincallback: INFO epoch: 6 - val_loss: 12.755282974243164 val_sparse_categorical_accuracy: 0.0025 val_sparse_categorical_crossentropy: 6.97173378944397 val_mean_anwer_guess: 526.03 val_sparse_top_2_categorical_accuracy: 0.005 val_sparse_top_5_categorical_accuracy: 0.01 val_sparse_top_10_categorical_accuracy: 0.0225 loss: 12.758541065454484 sparse_categorical_accuracy: 0.00234375 sparse_categorical_crossentropy: 6.807421277463436 mean_anwer_guess: 429.9875 sparse_top_2_categorical_accuracy: 0.00546875 sparse_top_5_categorical_accuracy: 0.0125 sparse_top_10_categorical_accuracy: 0.02109375 lr: 0.001
Epoch 8/100000000 320/320 [==============================] - 794s 2s/step - loss: 12.7336 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.8156 - mean_anwer_guess: 451.8687 - sparse_top_2_categorical_accuracy: 0.0016 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0148 - val_loss: 12.7757 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9916 - val_mean_anwer_guess: 541.2100 - val_sparse_top_2_categorical_accuracy: 0.0025 - val_sparse_top_5_categorical_accuracy: 0.0050 - val_sparse_top_10_categorical_accuracy: 0.0100
traincallback: INFO epoch: 7 - val_loss: 12.775742168426513 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.991552557945251 val_mean_anwer_guess: 541.21 val_sparse_top_2_categorical_accuracy: 0.0025 val_sparse_top_5_categorical_accuracy: 0.005 val_sparse_top_10_categorical_accuracy: 0.01 loss: 12.733552849292755 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.815575960278511 mean_anwer_guess: 451.86875 sparse_top_2_categorical_accuracy: 0.0015625 sparse_top_5_categorical_accuracy: 0.00859375 sparse_top_10_categorical_accuracy: 0.01484375 lr: 0.001
Epoch 9/100000000 320/320 [==============================] - 786s 2s/step - loss: 12.7661 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 6.8123 - mean_anwer_guess: 441.0469 - sparse_top_2_categorical_accuracy: 0.0023 - sparse_top_5_categorical_accuracy: 0.0055 - sparse_top_10_categorical_accuracy: 0.0156 - val_loss: 12.7747 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9789 - val_mean_anwer_guess: 501.0600 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0000e+00 - val_sparse_top_10_categorical_accuracy: 0.0050
traincallback: INFO epoch: 8 - val_loss: 12.774747228622436 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.978899388313294 val_mean_anwer_guess: 501.06 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0 val_sparse_top_10_categorical_accuracy: 0.005 loss: 12.766103932261467 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 6.812343089282512 mean_anwer_guess: 441.046875 sparse_top_2_categorical_accuracy: 0.00234375 sparse_top_5_categorical_accuracy: 0.00546875 sparse_top_10_categorical_accuracy: 0.015625 lr: 0.001
Epoch 10/100000000 320/320 [==============================] - 781s 2s/step - loss: 12.8401 - sparse_categorical_accuracy: 0.0023 - sparse_categorical_crossentropy: 6.8295 - mean_anwer_guess: 439.9531 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0078 - sparse_top_10_categorical_accuracy: 0.0164 - val_loss: 12.8246 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9870 - val_mean_anwer_guess: 507.5800 - val_sparse_top_2_categorical_accuracy: 0.0025 - val_sparse_top_5_categorical_accuracy: 0.0050 - val_sparse_top_10_categorical_accuracy: 0.0075
traincallback: INFO epoch: 9 - val_loss: 12.824644756317138 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.986988925933838 val_mean_anwer_guess: 507.58 val_sparse_top_2_categorical_accuracy: 0.0025 val_sparse_top_5_categorical_accuracy: 0.005 val_sparse_top_10_categorical_accuracy: 0.0075 loss: 12.840075600147248 sparse_categorical_accuracy: 0.00234375 sparse_categorical_crossentropy: 6.829490113258362 mean_anwer_guess: 439.953125 sparse_top_2_categorical_accuracy: 0.00390625 sparse_top_5_categorical_accuracy: 0.0078125 sparse_top_10_categorical_accuracy: 0.01640625 lr: 0.001
Epoch 11/100000000 320/320 [==============================] - 769s 2s/step - loss: 12.7090 - sparse_categorical_accuracy: 0.0016 - sparse_categorical_crossentropy: 6.7856 - mean_anwer_guess: 438.3531 - sparse_top_2_categorical_accuracy: 0.0023 - sparse_top_5_categorical_accuracy: 0.0086 - sparse_top_10_categorical_accuracy: 0.0172 - val_loss: 12.6598 - val_sparse_categorical_accuracy: 0.0025 - val_sparse_categorical_crossentropy: 6.9976 - val_mean_anwer_guess: 535.0700 - val_sparse_top_2_categorical_accuracy: 0.0025 - val_sparse_top_5_categorical_accuracy: 0.0050 - val_sparse_top_10_categorical_accuracy: 0.0075
traincallback: INFO epoch: 10 - val_loss: 12.659755830764771 val_sparse_categorical_accuracy: 0.0025 val_sparse_categorical_crossentropy: 6.997620878219604 val_mean_anwer_guess: 535.07 val_sparse_top_2_categorical_accuracy: 0.0025 val_sparse_top_5_categorical_accuracy: 0.005 val_sparse_top_10_categorical_accuracy: 0.0075 loss: 12.708958914875984 sparse_categorical_accuracy: 0.0015625 sparse_categorical_crossentropy: 6.785642342269421 mean_anwer_guess: 438.353125 sparse_top_2_categorical_accuracy: 0.00234375 sparse_top_5_categorical_accuracy: 0.00859375 sparse_top_10_categorical_accuracy: 0.0171875 lr: 0.001
Epoch 12/100000000 320/320 [==============================] - 783s 2s/step - loss: 12.7437 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 6.7799 - mean_anwer_guess: 440.8656 - sparse_top_2_categorical_accuracy: 0.0016 - sparse_top_5_categorical_accuracy: 0.0070 - sparse_top_10_categorical_accuracy: 0.0148 - val_loss: 12.8805 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 7.0201 - val_mean_anwer_guess: 497.7000 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0050 - val_sparse_top_10_categorical_accuracy: 0.0100
traincallback: INFO epoch: 11 - val_loss: 12.880471305847168 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 7.020058689117431 val_mean_anwer_guess: 497.7 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.005 val_sparse_top_10_categorical_accuracy: 0.01 loss: 12.743705308437347 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 6.77988765835762 mean_anwer_guess: 440.865625 sparse_top_2_categorical_accuracy: 0.0015625 sparse_top_5_categorical_accuracy: 0.00703125 sparse_top_10_categorical_accuracy: 0.01484375 lr: 0.001
Epoch 13/100000000 319/320 [============================>.] - ETA: 2s - loss: 12.8122 - sparse_categorical_accuracy: 7.8370e-04 - sparse_categorical_crossentropy: 6.7959 - mean_anwer_guess: 490.0690 - sparse_top_2_categorical_accuracy: 0.0016 - sparse_top_5_categorical_accuracy: 0.0063 - sparse_top_10_categorical_accuracy: 0.0172
root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3030] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3030] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3031] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3031] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3032] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3032] is not in the [index]'
320/320 [==============================] - 769s 2s/step - loss: 12.8128 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 6.7966 - mean_anwer_guess: 490.5000 - sparse_top_2_categorical_accuracy: 0.0016 - sparse_top_5_categorical_accuracy: 0.0063 - sparse_top_10_categorical_accuracy: 0.0172 - val_loss: 12.9069 - val_sparse_categorical_accuracy: 0.0025 - val_sparse_categorical_crossentropy: 6.9953 - val_mean_anwer_guess: 468.6600 - val_sparse_top_2_categorical_accuracy: 0.0050 - val_sparse_top_5_categorical_accuracy: 0.0125 - val_sparse_top_10_categorical_accuracy: 0.0175
traincallback: INFO epoch: 12 - val_loss: 12.906928291320801 val_sparse_categorical_accuracy: 0.0025 val_sparse_categorical_crossentropy: 6.995328159332275 val_mean_anwer_guess: 468.66 val_sparse_top_2_categorical_accuracy: 0.005 val_sparse_top_5_categorical_accuracy: 0.0125 val_sparse_top_10_categorical_accuracy: 0.0175 loss: 12.812796035408974 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 6.796648205816746 mean_anwer_guess: 490.5 sparse_top_2_categorical_accuracy: 0.0015625 sparse_top_5_categorical_accuracy: 0.00625 sparse_top_10_categorical_accuracy: 0.0171875 lr: 0.001
Epoch 14/100000000 320/320 [==============================] - 796s 2s/step - loss: 12.8612 - sparse_categorical_accuracy: 7.8125e-04 - sparse_categorical_crossentropy: 6.7987 - mean_anwer_guess: 458.7531 - sparse_top_2_categorical_accuracy: 0.0039 - sparse_top_5_categorical_accuracy: 0.0070 - sparse_top_10_categorical_accuracy: 0.0172 - val_loss: 13.0151 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 7.0287 - val_mean_anwer_guess: 462.7200 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0025 - val_sparse_top_10_categorical_accuracy: 0.0075
traincallback: INFO epoch: 13 - val_loss: 13.015073108673096 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 7.028693838119507 val_mean_anwer_guess: 462.72 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.0025 val_sparse_top_10_categorical_accuracy: 0.0075 loss: 12.86115721464157 sparse_categorical_accuracy: 0.00078125 sparse_categorical_crossentropy: 6.798652696609497 mean_anwer_guess: 458.753125 sparse_top_2_categorical_accuracy: 0.00390625 sparse_top_5_categorical_accuracy: 0.00703125 sparse_top_10_categorical_accuracy: 0.0171875 lr: 0.001
Epoch 00014: ReduceLROnPlateau reducing learning rate to 0.0006000000284984708. Epoch 15/100000000 320/320 [==============================] - 795s 2s/step - loss: 10.4306 - sparse_categorical_accuracy: 0.0023 - sparse_categorical_crossentropy: 6.7886 - mean_anwer_guess: 453.1719 - sparse_top_2_categorical_accuracy: 0.0031 - sparse_top_5_categorical_accuracy: 0.0055 - sparse_top_10_categorical_accuracy: 0.0211 - val_loss: 10.2737 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9637 - val_mean_anwer_guess: 490.0400 - val_sparse_top_2_categorical_accuracy: 0.0025 - val_sparse_top_5_categorical_accuracy: 0.0025 - val_sparse_top_10_categorical_accuracy: 0.0175
traincallback: INFO epoch: 14 - val_loss: 10.273672542572022 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.963716921806335 val_mean_anwer_guess: 490.04 val_sparse_top_2_categorical_accuracy: 0.0025 val_sparse_top_5_categorical_accuracy: 0.0025 val_sparse_top_10_categorical_accuracy: 0.0175 loss: 10.430625811219215 sparse_categorical_accuracy: 0.00234375 sparse_categorical_crossentropy: 6.788645024597645 mean_anwer_guess: 453.171875 sparse_top_2_categorical_accuracy: 0.003125 sparse_top_5_categorical_accuracy: 0.00546875 sparse_top_10_categorical_accuracy: 0.02109375 lr: 0.0006
Epoch 16/100000000 320/320 [==============================] - 784s 2s/step - loss: 10.3938 - sparse_categorical_accuracy: 0.0023 - sparse_categorical_crossentropy: 6.7822 - mean_anwer_guess: 432.9562 - sparse_top_2_categorical_accuracy: 0.0047 - sparse_top_5_categorical_accuracy: 0.0070 - sparse_top_10_categorical_accuracy: 0.0148 - val_loss: 10.3957 - val_sparse_categorical_accuracy: 0.0025 - val_sparse_categorical_crossentropy: 6.9853 - val_mean_anwer_guess: 480.5500 - val_sparse_top_2_categorical_accuracy: 0.0075 - val_sparse_top_5_categorical_accuracy: 0.0075 - val_sparse_top_10_categorical_accuracy: 0.0175
traincallback: INFO epoch: 15 - val_loss: 10.395728225708007 val_sparse_categorical_accuracy: 0.0025 val_sparse_categorical_crossentropy: 6.98533519744873 val_mean_anwer_guess: 480.55 val_sparse_top_2_categorical_accuracy: 0.0075 val_sparse_top_5_categorical_accuracy: 0.0075 val_sparse_top_10_categorical_accuracy: 0.0175 loss: 10.393761798739433 sparse_categorical_accuracy: 0.00234375 sparse_categorical_crossentropy: 6.782198789715767 mean_anwer_guess: 432.95625 sparse_top_2_categorical_accuracy: 0.0046875 sparse_top_5_categorical_accuracy: 0.00703125 sparse_top_10_categorical_accuracy: 0.01484375 lr: 0.0006
Epoch 17/100000000 320/320 [==============================] - 795s 2s/step - loss: 10.3923 - sparse_categorical_accuracy: 0.0031 - sparse_categorical_crossentropy: 6.7879 - mean_anwer_guess: 436.1344 - sparse_top_2_categorical_accuracy: 0.0031 - sparse_top_5_categorical_accuracy: 0.0117 - sparse_top_10_categorical_accuracy: 0.0172 - val_loss: 10.5543 - val_sparse_categorical_accuracy: 0.0025 - val_sparse_categorical_crossentropy: 6.9715 - val_mean_anwer_guess: 510.8900 - val_sparse_top_2_categorical_accuracy: 0.0025 - val_sparse_top_5_categorical_accuracy: 0.0025 - val_sparse_top_10_categorical_accuracy: 0.0050
traincallback: INFO epoch: 16 - val_loss: 10.554329509735107 val_sparse_categorical_accuracy: 0.0025 val_sparse_categorical_crossentropy: 6.971497311592102 val_mean_anwer_guess: 510.89 val_sparse_top_2_categorical_accuracy: 0.0025 val_sparse_top_5_categorical_accuracy: 0.0025 val_sparse_top_10_categorical_accuracy: 0.005 loss: 10.392290279269218 sparse_categorical_accuracy: 0.003125 sparse_categorical_crossentropy: 6.787858499586582 mean_anwer_guess: 436.134375 sparse_top_2_categorical_accuracy: 0.003125 sparse_top_5_categorical_accuracy: 0.01171875 sparse_top_10_categorical_accuracy: 0.0171875 lr: 0.0006
Epoch 18/100000000 320/320 [==============================] - 785s 2s/step - loss: 10.3798 - sparse_categorical_accuracy: 0.0039 - sparse_categorical_crossentropy: 6.7880 - mean_anwer_guess: 453.6187 - sparse_top_2_categorical_accuracy: 0.0078 - sparse_top_5_categorical_accuracy: 0.0117 - sparse_top_10_categorical_accuracy: 0.0187 - val_loss: 10.6460 - val_sparse_categorical_accuracy: 0.0000e+00 - val_sparse_categorical_crossentropy: 6.9940 - val_mean_anwer_guess: 533.2100 - val_sparse_top_2_categorical_accuracy: 0.0000e+00 - val_sparse_top_5_categorical_accuracy: 0.0050 - val_sparse_top_10_categorical_accuracy: 0.0100
root : INFO end date traincallback: INFO epoch: 17 - val_loss: 10.646020088195801 val_sparse_categorical_accuracy: 0.0 val_sparse_categorical_crossentropy: 6.994029064178466 val_mean_anwer_guess: 533.21 val_sparse_top_2_categorical_accuracy: 0.0 val_sparse_top_5_categorical_accuracy: 0.005 val_sparse_top_10_categorical_accuracy: 0.01 loss: 10.379760584235191 sparse_categorical_accuracy: 0.00390625 sparse_categorical_crossentropy: 6.788010680675507 mean_anwer_guess: 453.61875 sparse_top_2_categorical_accuracy: 0.0078125 sparse_top_5_categorical_accuracy: 0.01171875 sparse_top_10_categorical_accuracy: 0.01875 lr: 0.0006 traincallback: INFO training end: model_1
7:04:41.866862
uptime=datetime.datetime.now()-date_depart
logging.info("post train %s",uptime)
logging.info("remaining time %s",datetime.timedelta(hours=9)+date_depart-datetime.datetime.now())
root : INFO post train 7:04:44.719416 root : INFO remaining time 1:55:15.274879
if (datetime.datetime.now()-date_depart)<datetime.timedelta(hours=8,minutes=30):
fig=plt.figure(figsize=(15,7))
plt.subplot("211")
train_history=hist.history
histories=[hist_pre.history,hist.history,hist_post]
for k in train_history.keys():
train_history[k]=[]
for h in histories:
train_history[k]+=h.get(k,[])
for k in train_history.keys():
if "acc" in k:
plt.plot(train_history[k],label=k)
plt.ylim(ymin=0.95,ymax=1.0)
plt.legend()
plt.subplot("212")
plt.yscale("log")
for k in train_history.keys():
if "loss" in k:
plt.plot(train_history[k],label=k)
plt.legend()
plt.ylim(ymax=0.8)
fig.savefig("graph.png",dpi=200,transparent=False)
#IPython.display.Image(filename="graph.png")
--------------------------------------------------------------------------- NameError Traceback (most recent call last) <ipython-input-30-6e6b8d3c5db1> in <module>() 3 plt.subplot("211") 4 train_history=hist.history ----> 5 histories=[hist_pre.history,hist.history,hist_post] 6 for k in train_history.keys(): 7 train_history[k]=[] NameError: name 'hist_post' is not defined
if (datetime.datetime.now()-date_depart)<datetime.timedelta(hours=7,minutes=30):
for m,e in zip(model.metrics_names,
model.evaluate_generator(batch_gen(df_val,batch=64),steps=50, verbose=1) ):
print (m,e)
45/50 [==========================>...] - ETA: 12s
root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3030] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3030] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3031] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3031] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3032] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3032] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3033] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3033] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3034] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3034] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3035] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3035] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3036] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3036] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3037] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3037] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3038] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3038] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3039] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3039] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3040] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3040] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3041] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3041] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3042] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3042] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3043] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3043] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3044] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3044] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3045] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3045] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3046] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3046] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3047] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3047] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3048] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3048] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3049] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3049] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3050] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3050] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3051] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3051] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3052] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3052] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3053] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3053] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3054] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3054] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3055] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3055] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3056] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3056] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3057] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3057] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3058] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3058] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3059] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3059] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3060] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3060] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3061] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3061] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3062] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3062] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3063] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3063] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3064] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3064] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3065] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3065] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3066] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3066] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3067] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3067] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3068] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3068] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3069] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3069] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3070] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3070] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3071] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3071] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3072] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3072] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3073] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3073] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3074] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3074] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3075] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3075] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3076] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3076] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3077] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3077] is not in the [index]' root : ERROR imgaug_batch_gen Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1790, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3078] is not in the [index]' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<ipython-input-16-814ea07d8116>", line 27, in imgaug_batch_gen if df.loc[j+i,"req"] is None: File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1472, in __getitem__ return self._getitem_tuple(key) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 870, in _getitem_tuple return self._getitem_lowerdim(tup) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 998, in _getitem_lowerdim section = self._getitem_axis(key, axis=i) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1911, in _getitem_axis self._validate_key(key, axis) File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1798, in _validate_key error() File "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py", line 1785, in error axis=self.obj._get_axis_name(axis))) KeyError: 'the label [3078] is not in the [index]'
50/50 [==============================] - 127s 3s/step loss 12.688553867340088 sparse_categorical_accuracy 0.0009375 sparse_categorical_crossentropy 7.004177312850953 mean_anwer_guess 465.0 sparse_top_2_categorical_accuracy 0.001875 sparse_top_5_categorical_accuracy 0.004375 sparse_top_10_categorical_accuracy 0.0090625
if (datetime.datetime.now()-date_depart)<datetime.timedelta(hours=7,minutes=30):
ann_file = '../input/inaturalist-2019-fgvc6/test2019.json'
with open(ann_file) as data_file:
test_anns = json.load(data_file)
df_test=pd.DataFrame(test_anns['images'])[["id","file_name"]]
df_test
test_batch=50
if (datetime.datetime.now()-date_depart)<datetime.timedelta(hours=7,minutes=0):
test_datagen = ImageDataGenerator(rescale=1./255)
test_generator = test_datagen.flow_from_dataframe(
dataframe=df_test,
directory = "../input/inaturalist-2019-fgvc6/test2019",
x_col="file_name",
target_size = (dcrop,dcrop),
batch_size = test_batch,
shuffle = False,
class_mode = None
)
lengen=len(test_generator.filenames)
#math.gcd(lengen)
predict=model.predict_generator(test_generator, steps = len(test_generator.filenames)//test_batch+1,verbose=1)
sub=np.argsort(predict)[:lengen,-10:]
sub=np.flip(sub,1)
df_test["predicted"]=[" ".join(str(n) for n in pred )for pred in sub ]
df_test["preds"]=df_test["predicted"]
df_test[["id","preds", 'predicted']].to_csv("submission.csv", index=False)
df_test
uptime=datetime.datetime.now()-date_depart
logging.info("end %s",uptime)
logging.info("remaining time %s",datetime.timedelta(hours=9)+date_depart-datetime.datetime.now())
root : INFO end 7:06:57.295425 root : INFO remaining time 1:53:02.703319