diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000000000000000000000000000000000000..68da32b0340db619b8707e14175c407cc69e6266 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,13 @@ +[isort] +line_length = 100 +multi_line_output = 3 +balanced_wrapping = True +# skip = tools/test_net.py, tools/train_net.py +known_standard_library = setuptools +known_myself = official +known_data_processing = cv2,numpy,scipy,PIL,matplotlib +known_datasets = pycocotools +known_deeplearning = megengine +sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,data_processing,datasets,deeplearning,myself,LOCALFOLDER +no_lines_before=STDLIB,THIRDPARTY,datasets +default_section = FIRSTPARTY diff --git a/official/vision/detection/layers/basic/functional.py b/official/vision/detection/layers/basic/functional.py index 8fdff3f5c5db77f2ed12310960a8ba4c981fd674..36bcf7ff79b0d6c0d12fbe361d37401b4a90bdfa 100644 --- a/official/vision/detection/layers/basic/functional.py +++ b/official/vision/detection/layers/basic/functional.py @@ -6,10 +6,10 @@ # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -import megengine as mge -import megengine.functional as F import numpy as np +import megengine as mge +import megengine.functional as F from megengine.core import Tensor diff --git a/official/vision/detection/layers/basic/norm.py b/official/vision/detection/layers/basic/norm.py index 43a0917952761c048810bbcbfd883e68d9f080b5..073d758a24f2f13d1a96f1f32c5d4fd3567a6c83 100644 --- a/official/vision/detection/layers/basic/norm.py +++ b/official/vision/detection/layers/basic/norm.py @@ -24,9 +24,9 @@ # This file has been modified by Megvii ("Megvii Modifications"). # All Megvii Modifications are Copyright (C) 2014-2020 Megvii Inc. All rights reserved. # --------------------------------------------------------------------- -import megengine.module as M import numpy as np +import megengine.module as M from megengine.core import Buffer diff --git a/official/vision/detection/layers/det/anchor.py b/official/vision/detection/layers/det/anchor.py index 505553f753b987d0d4ddd78c5b27ba63cdea498c..83be5ac78b8cf51a8cf5580ed7d29e59f6ec2919 100644 --- a/official/vision/detection/layers/det/anchor.py +++ b/official/vision/detection/layers/det/anchor.py @@ -8,10 +8,10 @@ # "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. from abc import ABCMeta, abstractmethod -import megengine.functional as F import numpy as np -from megengine.core import tensor, Tensor +import megengine.functional as F +from megengine.core import Tensor, tensor class BaseAnchorGenerator(metaclass=ABCMeta): diff --git a/official/vision/detection/layers/det/box_utils.py b/official/vision/detection/layers/det/box_utils.py index 4d03f5e8b1e0ac86a15320f6ea5b065cf317df1e..23bbba622bfd5e6557b5be7fae3fe89beba5cd61 100644 --- a/official/vision/detection/layers/det/box_utils.py +++ b/official/vision/detection/layers/det/box_utils.py @@ -9,7 +9,6 @@ from abc import ABCMeta, abstractmethod import megengine.functional as F - from megengine.core import Tensor diff --git a/official/vision/detection/layers/det/loss.py b/official/vision/detection/layers/det/loss.py index b0fa4bf91f39e9ed7ab1f0ba333c634cebef0a35..3e75f29a2c65274424dc26887a67efe4ff097271 100644 --- a/official/vision/detection/layers/det/loss.py +++ b/official/vision/detection/layers/det/loss.py @@ -7,7 +7,6 @@ # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. import megengine.functional as F - from megengine.core import Tensor diff --git a/official/vision/detection/layers/det/pooler.py b/official/vision/detection/layers/det/pooler.py index d415b520a06676624d6645939b061deead331965..a3456ecaf9a71a25f8db2e964aa0996b6e88570a 100644 --- a/official/vision/detection/layers/det/pooler.py +++ b/official/vision/detection/layers/det/pooler.py @@ -9,6 +9,7 @@ import math import numpy as np + import megengine as mge import megengine.functional as F diff --git a/official/vision/detection/layers/det/retinanet.py b/official/vision/detection/layers/det/retinanet.py index df6d195685124a520de28a72cb1959e754a3b917..88c8c6e2fe92ed82507d2677d9ead8ec1beb8ab2 100644 --- a/official/vision/detection/layers/det/retinanet.py +++ b/official/vision/detection/layers/det/retinanet.py @@ -10,7 +10,6 @@ import math from typing import List import megengine.module as M - from megengine.core import Tensor from official.vision.detection.layers import basic diff --git a/official/vision/detection/layers/det/rpn.py b/official/vision/detection/layers/det/rpn.py index dfce3470465faab8b23663964dd8a09eaf5fe37d..3c0aebe7ed4da359512751803d48c716b21e9987 100644 --- a/official/vision/detection/layers/det/rpn.py +++ b/official/vision/detection/layers/det/rpn.py @@ -7,9 +7,10 @@ # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. import megengine as mge -import megengine.random as rand import megengine.functional as F import megengine.module as M +import megengine.random as rand + from official.vision.detection import layers from official.vision.detection.tools.gpu_nms import batched_nms diff --git a/official/vision/detection/models/faster_rcnn_fpn.py b/official/vision/detection/models/faster_rcnn_fpn.py index eaaff9a7d51ecb41b7e972d7a0f849b07a214a9b..e2aa411ef126641603b2b6b1bf5960366e5770a7 100644 --- a/official/vision/detection/models/faster_rcnn_fpn.py +++ b/official/vision/detection/models/faster_rcnn_fpn.py @@ -7,6 +7,7 @@ # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. import numpy as np + import megengine as mge import megengine.functional as F import megengine.module as M diff --git a/official/vision/detection/models/retinanet.py b/official/vision/detection/models/retinanet.py index f576c4bc67b69267f60a606b92aa6f27e066defa..e509239421a53685a0784a63313ad895a68b3777 100644 --- a/official/vision/detection/models/retinanet.py +++ b/official/vision/detection/models/retinanet.py @@ -6,10 +6,11 @@ # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +import numpy as np + import megengine as mge import megengine.functional as F import megengine.module as M -import numpy as np from official.vision.classification.resnet.model import resnet50 from official.vision.detection import layers diff --git a/official/vision/detection/tools/gpu_nms.py b/official/vision/detection/tools/gpu_nms.py index cf8422917565721b8a89114e2252bd27744b2c0e..4a3ee3ff98aefd6a7c2b4091cd98f6e2568c7397 100644 --- a/official/vision/detection/tools/gpu_nms.py +++ b/official/vision/detection/tools/gpu_nms.py @@ -5,6 +5,7 @@ import os import struct import numpy as np + import megengine as mge import megengine.functional as F from megengine._internal.craniotome import CraniotomeBase diff --git a/official/vision/detection/tools/inference.py b/official/vision/detection/tools/inference.py index c1a35560cd4f3b54db17d80febbc30a02c9062df..3bc53f369d1b29fcb5592835135db123d233c415 100644 --- a/official/vision/detection/tools/inference.py +++ b/official/vision/detection/tools/inference.py @@ -12,8 +12,9 @@ import os import sys import cv2 -import megengine as mge import numpy as np + +import megengine as mge from megengine import jit from megengine.data.dataset import COCO diff --git a/official/vision/detection/tools/test.py b/official/vision/detection/tools/test.py index 4fe87a5157096a06753a9220c4e02f16b702966d..5cf46d0bec344da2f2268df030fb1b6f83affd6c 100644 --- a/official/vision/detection/tools/test.py +++ b/official/vision/detection/tools/test.py @@ -13,13 +13,14 @@ import os import random import sys from multiprocessing import Process, Queue +from tqdm import tqdm import cv2 -import megengine as mge import numpy as np + +import megengine as mge from megengine import jit from megengine.data import DataLoader, SequentialSampler -from tqdm import tqdm from official.vision.detection.tools.data_mapper import data_mapper from official.vision.detection.tools.nms import py_cpu_nms diff --git a/official/vision/detection/tools/train.py b/official/vision/detection/tools/train.py index fb280c54ce3e6c6178c82a1486aa275ea73be891..d842e61c4cac7ddb0f1304031064af123657a118 100644 --- a/official/vision/detection/tools/train.py +++ b/official/vision/detection/tools/train.py @@ -14,16 +14,18 @@ import importlib import multiprocessing as mp import os import sys +import time from collections import defaultdict +from tabulate import tabulate -import megengine as mge import numpy as np + +import megengine as mge from megengine import distributed as dist from megengine import jit from megengine import optimizer as optim from megengine.data import Collator, DataLoader, Infinite, RandomSampler from megengine.data import transform as T -from tabulate import tabulate from official.vision.detection.tools.data_mapper import data_mapper @@ -81,7 +83,8 @@ def worker(rank, world_size, args): weights = mge.load(args.weight_file) model.backbone.bottom_up.load_state_dict(weights) - logger.info("Prepare dataset") + if rank == 0: + logger.info("Prepare dataset") loader = build_dataloader(model.batch_size, args.dataset_dir, model.cfg) train_loader = iter(loader["train"]) @@ -155,21 +158,32 @@ def train_one_epoch( return losses meter = AverageMeter(record_len=model.cfg.num_losses) + time_meter = AverageMeter(record_len=2) log_interval = model.cfg.log_interval for step in range(tot_steps): adjust_learning_rate(opt, epoch_id, step, model, world_size) + + data_tik = time.time() mini_batch = next(data_queue) + data_tok = time.time() + model.inputs["image"].set_value(mini_batch["data"]) model.inputs["gt_boxes"].set_value(mini_batch["gt_boxes"]) model.inputs["im_info"].set_value(mini_batch["im_info"]) + tik = time.time() opt.zero_grad() loss_list = propagate() opt.step() + tok = time.time() + + time_meter.update([tok - tik, data_tok - data_tik]) if rank == 0: + info_str = "e%d, %d/%d, lr:%f, " loss_str = ", ".join(["{}:%f".format(loss) for loss in model.cfg.losses_keys]) - log_info_str = "e%d, %d/%d, lr:%f, " + loss_str + time_str = ", train_time:%.3fs, data_time:%.3fs" + log_info_str = info_str + loss_str + time_str meter.update([loss.numpy() for loss in loss_list]) if step % log_interval == 0: average_loss = meter.average() @@ -180,8 +194,10 @@ def train_one_epoch( tot_steps, opt.param_groups[0]["lr"], *average_loss, + *time_meter.average() ) meter.reset() + time_meter.reset() def make_parser():