New image resizing mode: random crops.

config.IMAGE_RESIZE_MODE = "crop" can be used in
training only to pick random crops from training
images.

mrcnn/config.py

@@ -99,13 +99,22 @@ class Config(object):
     # square: Resize and pad with zeros to get a square image
     #         of size [max_dim, max_dim].
     # pad64:  Pads width and height with zeros to make them multiples of 64.
-    #         If IMAGE_MIN_DIM is not None, then scale the small side to
-    #         that size before padding. IMAGE_MAX_DIM is ignored in this mode.
+    #         If IMAGE_MIN_DIM or IMAGE_MIN_SCALE are not None, then it scales
+    #         up before padding. IMAGE_MAX_DIM is ignored in this mode.
     #         The multiple of 64 is needed to ensure smooth scaling of feature
     #         maps up and down the 6 levels of the FPN pyramid (2**6=64).
+    # crop:   Picks random crops from the image. First, scales the image based
+    #         on IMAGE_MIN_DIM and IMAGE_MIN_SCALE, then picks a random crop of
+    #         size IMAGE_MIN_DIM x IMAGE_MIN_DIM. Can be used in training only.
+    #         IMAGE_MAX_DIM is not used in this mode.
     IMAGE_RESIZE_MODE = "square"
     IMAGE_MIN_DIM = 800
     IMAGE_MAX_DIM = 1024
+    # Minimum scaling ratio. Checked after MIN_IMAGE_DIM and can force further
+    # up scaling. For example, if set to 2 then images are scaled up to double
+    # the width and height, or more, even if MIN_IMAGE_DIM doesn't require it.
+    # Howver, in 'square' mode, it can be overruled by IMAGE_MAX_DIM.
+    IMAGE_MIN_SCALE = 0
 
     # Image mean (RGB)
     MEAN_PIXEL = np.array([123.7, 116.8, 103.9])
@@ -177,8 +186,10 @@ class Config(object):
         self.BATCH_SIZE = self.IMAGES_PER_GPU * self.GPU_COUNT
 
         # Input image size
-        self.IMAGE_SHAPE = np.array(
-            [self.IMAGE_MAX_DIM, self.IMAGE_MAX_DIM, 3])
+        if self.IMAGE_RESIZE_MODE == "crop":
+            self.IMAGE_SHAPE = np.array([self.IMAGE_MIN_DIM, self.IMAGE_MIN_DIM, 3])
+        else:
+            self.IMAGE_SHAPE = np.array([self.IMAGE_MAX_DIM, self.IMAGE_MAX_DIM, 3])
 
         # Image meta data length
         # See compose_image_meta() for details

mrcnn/model.py

@@ -1206,12 +1206,13 @@ def load_image_gt(dataset, config, image_id, augment=False, augmentation=None,
     image = dataset.load_image(image_id)
     mask, class_ids = dataset.load_mask(image_id)
     original_shape = image.shape
-    image, window, scale, padding = utils.resize_image(
+    image, window, scale, padding, crop = utils.resize_image(
         image,
         min_dim=config.IMAGE_MIN_DIM,
+        min_scale=config.IMAGE_MIN_SCALE,
         max_dim=config.IMAGE_MAX_DIM,
         mode=config.IMAGE_RESIZE_MODE)
-    mask = utils.resize_mask(mask, scale, padding)
+    mask = utils.resize_mask(mask, scale, padding, crop)
 
     # Random horizontal flips.
     # TODO: will be removed in a future update in favor of augmentation
@@ -2335,9 +2336,10 @@ class MaskRCNN():
         for image in images:
             # Resize image
             # TODO: move resizing to mold_image()
-            molded_image, window, scale, padding = utils.resize_image(
+            molded_image, window, scale, padding, crop = utils.resize_image(
                 image,
                 min_dim=self.config.IMAGE_MIN_DIM,
+                min_scale=self.config.IMAGE_MIN_SCALE,
                 max_dim=self.config.IMAGE_MAX_DIM,
                 mode=self.config.IMAGE_RESIZE_MODE)
             molded_image = mold_image(molded_image, self.config)

mrcnn/utils.py

@@ -386,22 +386,28 @@ class Dataset(object):
         return mask, class_ids
 
 
-def resize_image(image, min_dim=None, max_dim=None, mode="square"):
+def resize_image(image, min_dim=None, max_dim=None, min_scale=None, mode="square"):
     """Resizes an image keeping the aspect ratio unchanged.
 
     min_dim: if provided, resizes the image such that it's smaller
         dimension == min_dim
     max_dim: if provided, ensures that the image longest side doesn't
         exceed this value.
+    min_scale: if provided, ensure that the image is scaled up by at least
+        this percent even if min_dim doesn't require it.
     mode: Resizing mode.
         none: No resizing. Return the image unchanged.
         square: Resize and pad with zeros to get a square image
             of size [max_dim, max_dim].
         pad64: Pads width and height with zeros to make them multiples of 64.
-               If min_dim is provided, it scales the small side to >= min_dim
+               If min_dim or min_scale are provided, it scales the image up
                before padding. max_dim is ignored in this mode.
                The multiple of 64 is needed to ensure smooth scaling of feature
                maps up and down the 6 levels of the FPN pyramid (2**6=64).
+        crop: Picks random crops from the image. First, scales the image based
+              on min_dim and min_scale, then picks a random crop of
+              size min_dim x min_dim. Can be used in training only.
+              max_dim is not used in this mode.
 
     Returns:
     image: the resized image
@@ -419,14 +425,18 @@ def resize_image(image, min_dim=None, max_dim=None, mode="square"):
     window = (0, 0, h, w)
     scale = 1
     padding = [(0, 0), (0, 0), (0, 0)]
+    crop = None
 
     if mode == "none":
-        return image, window, scale, padding
+        return image, window, scale, padding, crop
 
     # Scale?
     if min_dim:
         # Scale up but not down
         scale = max(1, min_dim / min(h, w))
+    if scale < min_scale:
+        scale = min_scale
+
     # Does it exceed max dim?
     if max_dim and mode == "square":
         image_max = max(h, w)
@@ -438,7 +448,8 @@ def resize_image(image, min_dim=None, max_dim=None, mode="square"):
         image = skimage.transform.resize(
             image, (round(h * scale), round(w * scale)),
             order=1, mode="constant", preserve_range=True)
-    # Need padding?
+
+    # Need padding or cropping?
     if mode == "square":
         # Get new height and width
         h, w = image.shape[:2]
@@ -470,12 +481,20 @@ def resize_image(image, min_dim=None, max_dim=None, mode="square"):
         padding = [(top_pad, bottom_pad), (left_pad, right_pad), (0, 0)]
         image = np.pad(image, padding, mode='constant', constant_values=0)
         window = (top_pad, left_pad, h + top_pad, w + left_pad)
+    elif mode == "crop":
+        # Pick a random crop
+        h, w = image.shape[:2]
+        y = random.randint(0, (h - min_dim))
+        x = random.randint(0, (w - min_dim))
+        crop = (y, x, min_dim, min_dim)
+        image = image[y:y + min_dim, x:x + min_dim]
+        window = (0, 0, min_dim, min_dim)
     else:
         raise Exception("Mode {} not supported".format(mode))
-    return image.astype(image_dtype), window, scale, padding
+    return image.astype(image_dtype), window, scale, padding, crop
 
 
-def resize_mask(mask, scale, padding):
+def resize_mask(mask, scale, padding, crop=None):
     """Resizes a mask using the given scale and padding.
     Typically, you get the scale and padding from resize_image() to
     ensure both, the image and the mask, are resized consistently.
@@ -489,7 +508,11 @@ def resize_mask(mask, scale, padding):
     with warnings.catch_warnings():
         warnings.simplefilter("ignore")
         mask = scipy.ndimage.zoom(mask, zoom=[scale, scale, 1], order=0)
-    mask = np.pad(mask, padding, mode='constant', constant_values=0)
+    if crop is not None:
+        y, x, h, w = crop
+        mask = mask[y:y + h, x:x + w]
+    else:
+        mask = np.pad(mask, padding, mode='constant', constant_values=0)
     return mask
 
 

samples/balloon/inspect_balloon_data.ipynb

@@ -279,7 +279,7 @@
     "mask, class_ids = dataset.load_mask(image_id)\n",
     "original_shape = image.shape\n",
     "# Resize\n",
-    "image, window, scale, padding = utils.resize_image(\n",
+    "image, window, scale, padding, _ = utils.resize_image(\n",
     "    image, \n",
     "    min_dim=config.IMAGE_MIN_DIM, \n",
     "    max_dim=config.IMAGE_MAX_DIM,\n",

samples/coco/inspect_data.ipynb

@@ -376,7 +376,7 @@
     "mask, class_ids = dataset.load_mask(image_id)\n",
     "original_shape = image.shape\n",
     "# Resize\n",
-    "image, window, scale, padding = utils.resize_image(\n",
+    "image, window, scale, padding, _ = utils.resize_image(\n",
     "    image, \n",
     "    min_dim=config.IMAGE_MIN_DIM, \n",
     "    max_dim=config.IMAGE_MAX_DIM,\n",