Patch scripts for training dropblock resnet

Author: datumbox

references/classification/README.md

@@ -53,6 +53,13 @@ torchrun --nproc_per_node=8 train.py --model $MODEL
 
 Here `$MODEL` is one of `resnet18`, `resnet34`, `resnet50`, `resnet101` or `resnet152`.
 
+### ResNet with dropblock
+```
+torchrun --nproc_per_node=8 train.py --model resnet50 -b 128 --lr 0.4 --epochs 270
+```
+
+
+
 ### ResNext
 ```
 torchrun --nproc_per_node=8 train.py\

references/classification/train.py

@@ -288,6 +288,7 @@ def main(args):
             f"Invalid lr scheduler '{args.lr_scheduler}'. Only StepLR, CosineAnnealingLR and ExponentialLR "
             "are supported."
         )
+    main_lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[125, 200, 250], gamma=0.1)
 
     if args.lr_warmup_epochs > 0:
         if args.lr_warmup_method == "linear":

torchvision/models/resnet.py

@@ -6,6 +6,7 @@
 
 from .._internally_replaced_utils import load_state_dict_from_url
 from ..utils import _log_api_usage_once
+from ..ops import DropBlock2d
 
 
 __all__ = [
@@ -122,6 +123,7 @@ def __init__(
         base_width: int = 64,
         dilation: int = 1,
         norm_layer: Optional[Callable[..., nn.Module]] = None,
+        p: float = 0.0,
     ) -> None:
         super().__init__()
         if norm_layer is None:
@@ -130,31 +132,40 @@ def __init__(
         # Both self.conv2 and self.downsample layers downsample the input when stride != 1
         self.conv1 = conv1x1(inplanes, width)
         self.bn1 = norm_layer(width)
+        # we won't be doing scheduled p
+        self.drop1 = DropBlock2d(p, 7)
         self.conv2 = conv3x3(width, width, stride, groups, dilation)
         self.bn2 = norm_layer(width)
+        self.drop2 = DropBlock2d(p, 7)
         self.conv3 = conv1x1(width, planes * self.expansion)
         self.bn3 = norm_layer(planes * self.expansion)
+        self.drop3 = DropBlock2d(p, 7)
         self.relu = nn.ReLU(inplace=True)
         self.downsample = downsample
+        self.drop4 = DropBlock2d(p, 7)
         self.stride = stride
 
     def forward(self, x: Tensor) -> Tensor:
         identity = x
-
+        # as in https://github.com/tensorflow/tpu/blob/b24729de804fdb751b06467d3dce0637fa652060/models/official/resnet/resnet_model.py#L545-L579
         out = self.conv1(x)
         out = self.bn1(out)
         out = self.relu(out)
+        out = self.drop1(out)
 
         out = self.conv2(out)
         out = self.bn2(out)
         out = self.relu(out)
+        out = self.drop2(out)
 
         out = self.conv3(out)
         out = self.bn3(out)
+        out = self.drop3(out)
 
         if self.downsample is not None:
             identity = self.downsample(x)
 
+        identity = self.drop4(identity)
         out += identity
         out = self.relu(out)
 
@@ -198,8 +209,9 @@ def __init__(
         self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
         self.layer1 = self._make_layer(block, 64, layers[0])
         self.layer2 = self._make_layer(block, 128, layers[1], stride=2, dilate=replace_stride_with_dilation[0])
-        self.layer3 = self._make_layer(block, 256, layers[2], stride=2, dilate=replace_stride_with_dilation[1])
-        self.layer4 = self._make_layer(block, 512, layers[3], stride=2, dilate=replace_stride_with_dilation[2])
+        # https://github.com/tensorflow/tpu/blob/b24729de804fdb751b06467d3dce0637fa652060/models/official/resnet/resnet_main.py#L393-L394
+        self.layer3 = self._make_layer(block, 256, layers[2], stride=2, dilate=replace_stride_with_dilation[1], p=0.1 / 4)
+        self.layer4 = self._make_layer(block, 512, layers[3], stride=2, dilate=replace_stride_with_dilation[2], p=0.1)
         self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
         self.fc = nn.Linear(512 * block.expansion, num_classes)
 
@@ -227,6 +239,7 @@ def _make_layer(
         blocks: int,
         stride: int = 1,
         dilate: bool = False,
+        p: float = 0.0,
     ) -> nn.Sequential:
         norm_layer = self._norm_layer
         downsample = None
@@ -243,7 +256,7 @@ def _make_layer(
         layers = []
         layers.append(
             block(
-                self.inplanes, planes, stride, downsample, self.groups, self.base_width, previous_dilation, norm_layer
+                self.inplanes, planes, stride, downsample, self.groups, self.base_width, previous_dilation, norm_layer, p
             )
         )
         self.inplanes = planes * block.expansion
@@ -256,6 +269,7 @@ def _make_layer(
                     base_width=self.base_width,
                     dilation=self.dilation,
                     norm_layer=norm_layer,
+                    p=p
                 )
             )