diff --git a/official/vision/detection/configs/faster_rcnn_res101_coco_2x_800size.py b/official/vision/detection/configs/faster_rcnn_res101_coco_2x_800size.py
index da6c3fd07fab58d51c6923b24807dc26ce1b3a17..a3dfcfde796343aacf57a60552ea3434e0d9ca38 100644
--- a/official/vision/detection/configs/faster_rcnn_res101_coco_2x_800size.py
+++ b/official/vision/detection/configs/faster_rcnn_res101_coco_2x_800size.py
@@ -33,7 +33,9 @@ def faster_rcnn_res101_coco_2x_800size(batch_size=1, **kwargs):
     `"FPN" <https://arxiv.org/abs/1612.03144>`_
     `"COCO" <https://arxiv.org/abs/1405.0312>`_
     """
-    return models.FasterRCNN(CustomFasterRCNNConfig(), batch_size=batch_size, **kwargs)
+    cfg = CustomFasterRCNNConfig()
+    cfg.backbone_pretrained = False
+    return models.FasterRCNN(cfg, batch_size=batch_size, **kwargs)
 
 
 Net = models.FasterRCNN
diff --git a/official/vision/detection/configs/faster_rcnn_res50_coco_1x_800size.py b/official/vision/detection/configs/faster_rcnn_res50_coco_1x_800size.py
index 6eff2fe985293c9bb211b22b7b72aaaf0355e3bf..702a939d7068a128b966b2697410f4233ff95221 100644
--- a/official/vision/detection/configs/faster_rcnn_res50_coco_1x_800size.py
+++ b/official/vision/detection/configs/faster_rcnn_res50_coco_1x_800size.py
@@ -22,7 +22,9 @@ def faster_rcnn_res50_coco_1x_800size(batch_size=1, **kwargs):
     `"FPN" <https://arxiv.org/abs/1612.03144>`_
     `"COCO" <https://arxiv.org/abs/1405.0312>`_
     """
-    return models.FasterRCNN(models.FasterRCNNConfig(), batch_size=batch_size, **kwargs)
+    cfg = models.FasterRCNNConfig()
+    cfg.backbone_pretrained = False
+    return models.FasterRCNN(cfg, batch_size=batch_size, **kwargs)
 
 
 Net = models.FasterRCNN
diff --git a/official/vision/detection/configs/faster_rcnn_res50_coco_1x_800size_syncbn.py b/official/vision/detection/configs/faster_rcnn_res50_coco_1x_800size_syncbn.py
index 9527c90247b7979185fc37ac06ba0bef9f241679..744cc22cd184f3c09eb93196bfaa553da3b7a95f 100644
--- a/official/vision/detection/configs/faster_rcnn_res50_coco_1x_800size_syncbn.py
+++ b/official/vision/detection/configs/faster_rcnn_res50_coco_1x_800size_syncbn.py
@@ -32,7 +32,9 @@ def faster_rcnn_res50_coco_1x_800size_syncbn(batch_size=1, **kwargs):
     `"COCO" <https://arxiv.org/abs/1405.0312>`_
     `"SyncBN" <https://arxiv.org/abs/1711.07240>`_
     """
-    return models.FasterRCNN(CustomFasterRCNNConfig(), batch_size=batch_size, **kwargs)
+    cfg = CustomFasterRCNNConfig()
+    cfg.backbone_pretrained = False
+    return models.FasterRCNN(cfg, batch_size=batch_size, **kwargs)
 
 
 Net = models.FasterRCNN
diff --git a/official/vision/detection/configs/faster_rcnn_resx101_coco_2x_800size.py b/official/vision/detection/configs/faster_rcnn_resx101_coco_2x_800size.py
index 974d2f0d8bba8f2b614da8f5c61c49588d4f8863..56798d225ecfeb0b6ea8b88595d021ac53c5e88c 100644
--- a/official/vision/detection/configs/faster_rcnn_resx101_coco_2x_800size.py
+++ b/official/vision/detection/configs/faster_rcnn_resx101_coco_2x_800size.py
@@ -33,7 +33,9 @@ def faster_rcnn_resx101_coco_2x_800size(batch_size=1, **kwargs):
     `"FPN" <https://arxiv.org/abs/1612.03144>`_
     `"COCO" <https://arxiv.org/abs/1405.0312>`_
     """
-    return models.FasterRCNN(CustomFasterRCNNConfig(), batch_size=batch_size, **kwargs)
+    cfg = CustomFasterRCNNConfig()
+    cfg.backbone_pretrained = False
+    return models.FasterRCNN(cfg, batch_size=batch_size, **kwargs)
 
 
 Net = models.FasterRCNN
diff --git a/official/vision/detection/configs/retinanet_res101_coco_2x_800size.py b/official/vision/detection/configs/retinanet_res101_coco_2x_800size.py
index 456f7059383e7e8959c14d49efdc4177d5d1250d..51230ac87c266710bcd9f30e33fe00e92983642a 100644
--- a/official/vision/detection/configs/retinanet_res101_coco_2x_800size.py
+++ b/official/vision/detection/configs/retinanet_res101_coco_2x_800size.py
@@ -33,7 +33,9 @@ def retinanet_res101_coco_2x_800size(batch_size=1, **kwargs):
     `"FPN" <https://arxiv.org/abs/1612.03144>`_
     `"COCO" <https://arxiv.org/abs/1405.0312>`_
     """
-    return models.RetinaNet(CustomRetinaNetConfig(), batch_size=batch_size, **kwargs)
+    cfg = CustomRetinaNetConfig()
+    cfg.backbone_pretrained = False
+    return models.RetinaNet(cfg, batch_size=batch_size, **kwargs)
 
 
 Net = models.RetinaNet
diff --git a/official/vision/detection/configs/retinanet_res50_coco_1x_800size.py b/official/vision/detection/configs/retinanet_res50_coco_1x_800size.py
index 3cb0ba4a862bd6dc80524f92290cd0374c2c5287..df9c693d4af676d4ec2008ca17494794f15c2bf1 100644
--- a/official/vision/detection/configs/retinanet_res50_coco_1x_800size.py
+++ b/official/vision/detection/configs/retinanet_res50_coco_1x_800size.py
@@ -22,7 +22,9 @@ def retinanet_res50_coco_1x_800size(batch_size=1, **kwargs):
     `"FPN" <https://arxiv.org/abs/1612.03144>`_
     `"COCO" <https://arxiv.org/abs/1405.0312>`_
     """
-    return models.RetinaNet(models.RetinaNetConfig(), batch_size=batch_size, **kwargs)
+    cfg = models.RetinaNetConfig()
+    cfg.backbone_pretrained = False
+    return models.RetinaNet(cfg, batch_size=batch_size, **kwargs)
 
 
 Net = models.RetinaNet
diff --git a/official/vision/detection/configs/retinanet_res50_coco_1x_800size_syncbn.py b/official/vision/detection/configs/retinanet_res50_coco_1x_800size_syncbn.py
index bfd31d29597712b4d89a1cb182bc1e620b9c7606..2e591db695583db3406532645144c40bc4d8c102 100644
--- a/official/vision/detection/configs/retinanet_res50_coco_1x_800size_syncbn.py
+++ b/official/vision/detection/configs/retinanet_res50_coco_1x_800size_syncbn.py
@@ -32,7 +32,9 @@ def retinanet_res50_coco_1x_800size_syncbn(batch_size=1, **kwargs):
     `"COCO" <https://arxiv.org/abs/1405.0312>`_
     `"SyncBN" <https://arxiv.org/abs/1711.07240>`_
     """
-    return models.RetinaNet(CustomRetinaNetConfig(), batch_size=batch_size, **kwargs)
+    cfg = CustomRetinaNetConfig()
+    cfg.backbone_pretrained = False
+    return models.RetinaNet(cfg, batch_size=batch_size, **kwargs)
 
 
 Net = models.RetinaNet
diff --git a/official/vision/detection/configs/retinanet_resx101_coco_2x_800size.py b/official/vision/detection/configs/retinanet_resx101_coco_2x_800size.py
index 38c53169f32076a50a4395e3ef734e352835f414..e72e016e1070cf6cf15bc30756f25ea095d9ff10 100644
--- a/official/vision/detection/configs/retinanet_resx101_coco_2x_800size.py
+++ b/official/vision/detection/configs/retinanet_resx101_coco_2x_800size.py
@@ -33,7 +33,9 @@ def retinanet_resx101_coco_2x_800size(batch_size=1, **kwargs):
     `"FPN" <https://arxiv.org/abs/1612.03144>`_
     `"COCO" <https://arxiv.org/abs/1405.0312>`_
     """
-    return models.RetinaNet(CustomRetinaNetConfig(), batch_size=batch_size, **kwargs)
+    cfg = CustomRetinaNetConfig()
+    cfg.backbone_pretrained = False
+    return models.RetinaNet(cfg, batch_size=batch_size, **kwargs)
 
 
 Net = models.RetinaNet