update benchmark

deploy/utils/predictor.py

@@ -59,7 +59,8 @@ class Predictor(object):
             config.enable_tensorrt_engine(
                 precision_mode=Config.Precision.Half
                 if args.use_fp16 else Config.Precision.Float32,
-                max_batch_size=args.batch_size)
+                max_batch_size=args.batch_size,
+                min_subgraph_size=30)
 
         config.enable_memory_optim()
         # use zero copy

tests/DarkNet53.txt

@@ -33,7 +33,7 @@ fpgm_export:null
 distill_export:null
 export1:null
 export2:null
-##
+infer_model_url:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/whole_chain/DarkNet53_inference.tar
 infer_model:../inference/
 infer_export:null
 infer_quant:Fasle
@@ -48,16 +48,4 @@ inference:python/predict_cls.py -c configs/inference_cls.yaml
 -o Global.infer_imgs:../dataset/ILSVRC2012/val
 -o Global.save_log_path:null
 -o Global.benchmark:True
-#
-#
-#
-#
-===========================pretrained_model===========================
-ResNet50_vd:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/ResNet50_vd_pretrained.pdparams   
-ResNeXt101_vd_64x4d:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ResNeXt101_vd_64x4d_pretrained.pdparams
-HRNet_W18_C:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/HRNet_W18_C_pretrained.pdparams
-MobileNetV3_large_x1_0:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/MobileNetV3_large_x1_0_pretrained.pdparams
-DarkNet53:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DarkNet53_pretrained.pdparams 
-MobileNetV1:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/MobileNetV1_pretrained.pdparams
-MobileNetV2:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/MobileNetV2_pretrained.pdparams 
-ShuffleNetV2_x1_0:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ShuffleNetV2_x1_0_pretrained.pdparams
+null:null

tests/HRNet_W18_C.txt

@@ -33,7 +33,7 @@ fpgm_export:null
 distill_export:null
 export1:null
 export2:null
-##
+inference_model_url:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/whole_chain/HRNet_W18_C_inference.tar
 infer_model:../inference/
 infer_export:null
 infer_quant:Fasle
@@ -48,16 +48,4 @@ inference:python/predict_cls.py -c configs/inference_cls.yaml
 -o Global.infer_imgs:../dataset/ILSVRC2012/val
 -o Global.save_log_path:null
 -o Global.benchmark:True
-#
-#
-#
-#
-===========================pretrained_model===========================
-ResNet50_vd:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/ResNet50_vd_pretrained.pdparams   
-ResNeXt101_vd_64x4d:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ResNeXt101_vd_64x4d_pretrained.pdparams
-HRNet_W18_C:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/HRNet_W18_C_pretrained.pdparams
-MobileNetV3_large_x1_0:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/MobileNetV3_large_x1_0_pretrained.pdparams
-DarkNet53:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DarkNet53_pretrained.pdparams 
-MobileNetV1:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/MobileNetV1_pretrained.pdparams
-MobileNetV2:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/MobileNetV2_pretrained.pdparams 
-ShuffleNetV2_x1_0:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ShuffleNetV2_x1_0_pretrained.pdparams
+null:null

tests/LeViT_128S.txt

+===========================train_params===========================
+model_name:LeViT_128S
+python:python3.7
+gpu_list:0|0,1
+-o Global.device:gpu
+-o Global.auto_cast:null
+-o Global.epochs:lite_train_infer=2|whole_train_infer=120
+-o Global.output_dir:./output/
+-o DataLoader.Train.sampler.batch_size:8
+-o Global.pretrained_model:null
+train_model_name:latest
+train_infer_img_dir:./dataset/ILSVRC2012/val
+null:null
+##
+trainer:norm_train
+norm_train:tools/train.py -c ppcls/configs/ImageNet/LeViT/LeViT_128S.yaml
+pact_train:null
+fpgm_train:null
+distill_train:null
+null:null
+null:null
+##
+===========================eval_params=========================== 
+eval:tools/eval.py -c ppcls/configs/ImageNet/LeViT/LeViT_128S.yaml
+null:null
+##
+===========================infer_params==========================
+-o Global.save_inference_dir:./inference
+-o Global.pretrained_model:
+norm_export:tools/export_model.py -c ppcls/configs/ImageNet/LeViT/LeViT_128S.yaml
+quant_export:null
+fpgm_export:null
+distill_export:null
+export1:null
+export2:null
+infer_model_url:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/whole_chain/LeViT_128S_inference.tar
+infer_model:../inference/
+infer_export:null
+infer_quant:Fasle
+inference:python/predict_cls.py -c configs/inference_cls.yaml
+-o Global.use_gpu:True|Fasle
+-o Global.enable_mkldnn:True|False
+-o Global.cpu_num_threads:1|6
+-o Global.batch_size:1
+-o Global.use_tensorrt:True|False
+-o Global.use_fp16:True|False
+-o Global.inference_model_dir:../inference
+-o Global.infer_imgs:../dataset/ILSVRC2012/val
+-o Global.save_log_path:null
+-o Global.benchmark:True
+null:null

tests/MobileNetV1.txt

@@ -33,7 +33,7 @@ fpgm_export:null
 distill_export:null
 export1:null
 export2:null
-##
+inference_model_url:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/whole_chain/MobileNetV1_inference.tar
 infer_model:../inference/
 infer_export:null
 infer_quant:Fasle
@@ -48,16 +48,4 @@ inference:python/predict_cls.py -c configs/inference_cls.yaml
 -o Global.infer_imgs:../dataset/ILSVRC2012/val
 -o Global.save_log_path:null
 -o Global.benchmark:True
-#
-#
-#
-#
-===========================pretrained_model===========================
-ResNet50_vd:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/ResNet50_vd_pretrained.pdparams   
-ResNeXt101_vd_64x4d:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ResNeXt101_vd_64x4d_pretrained.pdparams
-HRNet_W18_C:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/HRNet_W18_C_pretrained.pdparams
-MobileNetV3_large_x1_0:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/MobileNetV3_large_x1_0_pretrained.pdparams
-DarkNet53:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DarkNet53_pretrained.pdparams 
-MobileNetV1:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/MobileNetV1_pretrained.pdparams
-MobileNetV2:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/MobileNetV2_pretrained.pdparams 
-ShuffleNetV2_x1_0:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ShuffleNetV2_x1_0_pretrained.pdparams
+null:null

tests/MobileNetV2.txt

@@ -33,7 +33,7 @@ fpgm_export:null
 distill_export:null
 export1:null
 export2:null
-##
+infer_model_url:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/whole_chain/MobileNetV2_inference.tar
 infer_model:../inference/
 infer_export:null
 infer_quant:Fasle
@@ -48,16 +48,4 @@ inference:python/predict_cls.py -c configs/inference_cls.yaml
 -o Global.infer_imgs:../dataset/ILSVRC2012/val
 -o Global.save_log_path:null
 -o Global.benchmark:True
-#
-#
-#
-#
-===========================pretrained_model===========================
-ResNet50_vd:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/ResNet50_vd_pretrained.pdparams   
-ResNeXt101_vd_64x4d:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ResNeXt101_vd_64x4d_pretrained.pdparams
-HRNet_W18_C:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/HRNet_W18_C_pretrained.pdparams
-MobileNetV3_large_x1_0:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/MobileNetV3_large_x1_0_pretrained.pdparams
-DarkNet53:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DarkNet53_pretrained.pdparams 
-MobileNetV1:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/MobileNetV1_pretrained.pdparams
-MobileNetV2:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/MobileNetV2_pretrained.pdparams 
-ShuffleNetV2_x1_0:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ShuffleNetV2_x1_0_pretrained.pdparams
+null:null

tests/MobileNetV3_large_x1_0.txt

@@ -33,7 +33,7 @@ fpgm_export:deploy/slim/slim.py -m export -c ppcls/configs/slim/MobileNetV3_larg
 distill_export:null
 export1:null
 export2:null
-##
+inference_model_url:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/whole_chain/MobileNetV3_large_x1_0_inference.tar
 infer_model:../inference/
 infer_export:null
 infer_quant:Fasle
@@ -48,16 +48,4 @@ inference:python/predict_cls.py -c configs/inference_cls.yaml
 -o Global.infer_imgs:../dataset/ILSVRC2012/val
 -o Global.save_log_path:null
 -o Global.benchmark:True
-#
-#
-#
-#
-===========================pretrained_model===========================
-ResNet50_vd:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/ResNet50_vd_pretrained.pdparams   
-ResNeXt101_vd_64x4d:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ResNeXt101_vd_64x4d_pretrained.pdparams
-HRNet_W18_C:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/HRNet_W18_C_pretrained.pdparams
-MobileNetV3_large_x1_0:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/MobileNetV3_large_x1_0_pretrained.pdparams
-DarkNet53:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DarkNet53_pretrained.pdparams 
-MobileNetV1:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/MobileNetV1_pretrained.pdparams
-MobileNetV2:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/MobileNetV2_pretrained.pdparams 
-ShuffleNetV2_x1_0:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ShuffleNetV2_x1_0_pretrained.pdparams
+null:null

tests/ResNeXt101_vd_64x4d.txt

@@ -33,7 +33,7 @@ fpgm_export:null
 distill_export:null
 export1:null
 export2:null
-##
+inference_model_url:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/whole_chain/ResNeXt101_64x4d_inference.tar
 infer_model:../inference/
 infer_export:null
 infer_quant:Fasle
@@ -48,16 +48,4 @@ inference:python/predict_cls.py -c configs/inference_cls.yaml
 -o Global.infer_imgs:../dataset/ILSVRC2012/val
 -o Global.save_log_path:null
 -o Global.benchmark:True
-#
-#
-#
-#
-===========================pretrained_model===========================
-ResNet50_vd:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/ResNet50_vd_pretrained.pdparams   
-ResNeXt101_vd_64x4d:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ResNeXt101_vd_64x4d_pretrained.pdparams
-HRNet_W18_C:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/HRNet_W18_C_pretrained.pdparams
-MobileNetV3_large_x1_0:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/MobileNetV3_large_x1_0_pretrained.pdparams
-DarkNet53:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DarkNet53_pretrained.pdparams 
-MobileNetV1:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/MobileNetV1_pretrained.pdparams
-MobileNetV2:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/MobileNetV2_pretrained.pdparams 
-ShuffleNetV2_x1_0:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ShuffleNetV2_x1_0_pretrained.pdparams
+null:null

tests/ResNet50_vd.txt

 ===========================train_params===========================
 model_name:ResNet50_vd
 python:python3.7
-gpu_list:3,4
+gpu_list:0|0,1
 -o Global.device:gpu
 -o Global.auto_cast:null
 -o Global.epochs:lite_train_infer=2|whole_train_infer=120
@@ -33,7 +33,7 @@ fpgm_export:deploy/slim/slim.py -m export -c ppcls/configs/slim/ResNet50_vd_prun
 distill_export:null
 export1:null
 export2:null
-##
+infer_model_url:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/whole_chain/ResNet50_vd_inference.tar
 infer_model:../inference/
 infer_export:null
 infer_quant:Fasle
@@ -48,16 +48,4 @@ inference:python/predict_cls.py -c configs/inference_cls.yaml
 -o Global.infer_imgs:../dataset/ILSVRC2012/val
 -o Global.save_log_path:null
 -o Global.benchmark:True
-#
-#
-#
-#
-===========================pretrained_model===========================
-ResNet50_vd:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/ResNet50_vd_pretrained.pdparams   
-ResNeXt101_vd_64x4d:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ResNeXt101_vd_64x4d_pretrained.pdparams
-HRNet_W18_C:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/HRNet_W18_C_pretrained.pdparams
-MobileNetV3_large_x1_0:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/MobileNetV3_large_x1_0_pretrained.pdparams
-DarkNet53:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DarkNet53_pretrained.pdparams 
-MobileNetV1:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/MobileNetV1_pretrained.pdparams
-MobileNetV2:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/MobileNetV2_pretrained.pdparams 
-ShuffleNetV2_x1_0:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ShuffleNetV2_x1_0_pretrained.pdparams
+null:null

tests/ShuffleNetV2_x1_0.txt

@@ -33,7 +33,7 @@ fpgm_export:null
 distill_export:null
 export1:null
 export2:null
-##
+inference_model_url:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/whole_chain/ShuffleNetV2_x1_0_inference.tar
 infer_model:../inference/
 infer_export:null
 infer_quant:Fasle
@@ -48,16 +48,4 @@ inference:python/predict_cls.py -c configs/inference_cls.yaml
 -o Global.infer_imgs:../dataset/ILSVRC2012/val
 -o Global.save_log_path:null
 -o Global.benchmark:True
-#
-#
-#
-#
-===========================pretrained_model===========================
-ResNet50_vd:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/ResNet50_vd_pretrained.pdparams   
-ResNeXt101_vd_64x4d:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ResNeXt101_vd_64x4d_pretrained.pdparams
-HRNet_W18_C:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/HRNet_W18_C_pretrained.pdparams
-MobileNetV3_large_x1_0:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/MobileNetV3_large_x1_0_pretrained.pdparams
-DarkNet53:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DarkNet53_pretrained.pdparams 
-MobileNetV1:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/MobileNetV1_pretrained.pdparams
-MobileNetV2:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/MobileNetV2_pretrained.pdparams 
-ShuffleNetV2_x1_0:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ShuffleNetV2_x1_0_pretrained.pdparams
+null:null

tests/SwinTransformer_tiny_patch4_window7_224.txt

+===========================train_params===========================
+model_name:SwinTransformer_tiny_patch4_window7_224
+python:python3.7
+gpu_list:0|0,1
+-o Global.device:gpu
+-o Global.auto_cast:null
+-o Global.epochs:lite_train_infer=2|whole_train_infer=120
+-o Global.output_dir:./output/
+-o DataLoader.Train.sampler.batch_size:8
+-o Global.pretrained_model:null
+train_model_name:latest
+train_infer_img_dir:./dataset/ILSVRC2012/val
+null:null
+##
+trainer:norm_train
+norm_train:tools/train.py -c ppcls/configs/ImageNet/SwinTransformer/SwinTransformer_tiny_patch4_window7_224.yaml
+pact_train:null
+fpgm_train:null
+distill_train:null
+null:null
+null:null
+##
+===========================eval_params=========================== 
+eval:tools/eval.py -c ppcls/configs/ImageNet/SwinTransformer/SwinTransformer_tiny_patch4_window7_224.yaml
+null:null
+##
+===========================infer_params==========================
+-o Global.save_inference_dir:./inference
+-o Global.pretrained_model:
+norm_export:tools/export_model.py -c ppcls/configs/ImageNet/SwinTransformer/SwinTransformer_tiny_patch4_window7_224.yaml
+quant_export:null
+fpgm_export:null
+distill_export:null
+export1:null
+export2:null
+inference_model_url:https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/whole_chain/SwinTransformer_tiny_patch4_window7_224_inference.tar
+infer_model:../inference/
+infer_export:null
+infer_quant:Fasle
+inference:python/predict_cls.py -c configs/inference_cls.yaml
+-o Global.use_gpu:True|False
+-o Global.enable_mkldnn:True|False
+-o Global.cpu_num_threads:1|6
+-o Global.batch_size:1
+-o Global.use_tensorrt:True|False
+-o Global.use_fp16:True|False
+-o Global.inference_model_dir:../inference
+-o Global.infer_imgs:../dataset/ILSVRC2012/val
+-o Global.save_log_path:null
+-o Global.benchmark:True
+null:null

tests/prepare.sh

@@ -11,15 +11,21 @@ function func_parser_value(){
     strs=$1
     IFS=":"
     array=(${strs})
-    IFS="|"
-    tmp="${array[1]}:${array[2]}"
-    echo ${tmp}
+    if [ ${#array[*]} = 2 ]; then
+        echo ${array[1]}
+    else
+    	IFS="|"
+    	tmp="${array[1]}:${array[2]}"
+        echo ${tmp}
+    fi
 }
-inference_model_url=$(func_parser_value "${lines[50]}")
+model_name=$(func_parser_value "${lines[1]}")
+inference_model_url=$(func_parser_value "${lines[35]}")
 
 if [ ${MODE} = "lite_train_infer" ] || [ ${MODE} = "whole_infer" ];then
     # pretrain lite train data
     cd dataset
+    rm -rf ILSVRC2012
     wget -nc https://paddle-imagenet-models-name.bj.bcebos.com/data/whole_chain/whole_chain_little_train.tar
     tar xf whole_chain_little_train.tar
     ln -s whole_chain_little_train ILSVRC2012
@@ -30,18 +36,20 @@ if [ ${MODE} = "lite_train_infer" ] || [ ${MODE} = "whole_infer" ];then
 elif [ ${MODE} = "infer" ];then
     # download data
     cd dataset
+    rm -rf ILSVRC2012
     wget -nc https://paddle-imagenet-models-name.bj.bcebos.com/data/whole_chain/whole_chain_infer.tar
     tar xf whole_chain_infer.tar
     ln -s whole_chain_infer ILSVRC2012
     cd ILSVRC2012 
-    mv train.txt train_list.txt
     mv val.txt val_list.txt
     cd ../../
-   # download inference model
+    # download inference model
     eval "wget -nc $inference_model_url"
+    tar xf "${model_name}_inference.tar"
 
 elif [ ${MODE} = "whole_train_infer" ];then
     cd dataset
+    rm -rf ILSVRC2012
     wget -nc https://paddle-imagenet-models-name.bj.bcebos.com/data/whole_chain/whole_chain_CIFAR100.tar
     tar xf whole_chain_CIFAR100.tar
     ln -s whole_chain_CIFAR100 ILSVRC2012

tests/test.sh

@@ -227,6 +227,7 @@ if [ ${MODE} = "infer" ]; then
     IFS="|"
     infer_run_exports=(${infer_export_list})
     infer_quant_flag=(${infer_is_quant})
+    cd deploy
     for infer_model in ${infer_model_dir_list[*]}; do
         # run export
         if [ ${infer_run_exports[Count]} != "null" ];then
@@ -236,15 +237,16 @@ if [ ${MODE} = "infer" ]; then
             eval $export_cmd
             status_export=$?
             if [ ${status_export} = 0 ];then
-                status_check $status_export "${export_cmd}" "${status_log}"
+                status_check $status_export "${export_cmd}" "../${status_log}"
             fi
         fi
         #run inference
         is_quant=${infer_quant_flag[Count]}
         echo "is_quant: ${is_quant}"
-        func_inference "${python}" "${inference_py}" "${infer_model}" "${LOG_PATH}" "${infer_img_dir}" ${is_quant}
+        func_inference "${python}" "${inference_py}" "${infer_model}" "../${LOG_PATH}" "${infer_img_dir}" ${is_quant}
         Count=$(($Count + 1))
     done
+    cd ..
 
 else
     IFS="|"