Disable validation completely when overfit_batches>0

CHANGELOG.md

@@ -368,6 +368,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 ### Fixed
 
+- Fixed an issue that will cause non-default value of `Trainer`'s attributes `limit_train_batches`, `limit_val_batches` and `limit_test_batches` will be covered by another attribute `overfit_batches` when `overfit_batches` is larger than zero ([#9709](https://github.com/PyTorchLightning/pytorch-lightning/pull/9709))
 
 - Fixed an issue with logger outputs not being finalized correctly after prediction runs ([#8685](https://github.com/PyTorchLightning/pytorch-lightning/pull/8685))
 

pytorch_lightning/trainer/connectors/debugging_connector.py

@@ -78,9 +78,14 @@ def on_init_start(
     def determine_data_use_amount(self, overfit_batches: float) -> None:
         """Use less data for debugging purposes."""
         if overfit_batches > 0:
-            self.trainer.limit_train_batches = overfit_batches
-            self.trainer.limit_val_batches = overfit_batches
-            self.trainer.limit_test_batches = overfit_batches
+            # Avoid non-default value of ``limit_train/val/test_batches`` to be reset
+            # by overfit_batches
+            if self.trainer.limit_train_batches == 1.0:
+                self.trainer.limit_train_batches = overfit_batches
+            if self.trainer.limit_val_batches == 1.0:
+                self.trainer.limit_val_batches = overfit_batches
+            if self.trainer.limit_test_batches == 1.0:
+                self.trainer.limit_test_batches = overfit_batches
 
 
 def _determine_batch_limits(batches: Union[int, float], name: str) -> Union[int, float]:

tests/trainer/test_trainer.py

@@ -884,6 +884,18 @@ def validation_epoch_end(self, *args, **kwargs):
     assert not model.validation_step_invoked, "`validation_step` should not run when `limit_val_batches=0`"
     assert not model.validation_epoch_end_invoked, "`validation_epoch_end` should not run when `limit_val_batches=0`"
 
+    # check that limit_val_batches=0 turns off validation even when overfit_batches > 0
+    model = CurrentModel(**hparams)
+    trainer_options.update(overfit_batches=1)
+    trainer = Trainer(**trainer_options)
+    trainer.fit(model)
+
+    assert trainer.state.finished, f"Training failed with {trainer.state}"
+    assert trainer.current_epoch == 1
+    assert trainer.limit_val_batches == 0.0
+    assert not model.validation_step_invoked, "`validation_step` should not run when `limit_val_batches=0`"
+    assert not model.validation_epoch_end_invoked, "`validation_epoch_end` should not run when `limit_val_batches=0`"
+
     # check that limit_val_batches has no influence when fast_dev_run is turned on
     model = CurrentModel(**hparams)
     trainer_options.update(fast_dev_run=True)
@@ -896,6 +908,64 @@ def validation_epoch_end(self, *args, **kwargs):
     assert model.validation_epoch_end_invoked, "did not run `validation_epoch_end` with `fast_dev_run=True`"
 
 
+def test_irrevelance_between_non_default_overfit_batches_and_non_default_batch_limitation(tmpdir):
+    """Verify that when `overfit_batches` > 0,  `limit_train/val/test_batches` won't be resetted to
+    `overfit_batches` unless they are of default value."""
+    """Assure that non-default value of `limit_train/val/test_batches` won't be reset by `DebuggingConnector` when `overfit_batches` > 0"""
+
+    class CurrentModel(EvalModelTemplate):
+
+        validation_step_invoked = False
+        validation_epoch_end_invoked = False
+
+        def validation_step(self, *args, **kwargs):
+            self.validation_step_invoked = True
+            return super().validation_step(*args, **kwargs)
+
+        def validation_epoch_end(self, *args, **kwargs):
+            self.validation_epoch_end_invoked = True
+            return super().validation_epoch_end(*args, **kwargs)
+
+    hparams = EvalModelTemplate.get_default_hparams()
+    model = CurrentModel(**hparams)
+
+    trainer_options = dict(
+        default_root_dir=tmpdir,
+        enable_progress_bar=False,
+        max_epochs=2,
+        limit_train_batches=0.4,
+        limit_val_batches=0.0,
+        overfit_batches=1,
+        fast_dev_run=False,
+    )
+
+    trainer = Trainer(**trainer_options)
+    trainer.fit(model)
+
+    # check that limit_xxx_batches won't be reset when they are non-default value and overfit_batches > 0
+    assert trainer.state.finished, f"Training failed with {trainer.state}"
+    assert trainer.current_epoch == 1
+    assert trainer.limit_train_batches == 0.4
+    assert trainer.limit_val_batches == 0.0
+    assert not model.validation_step_invoked, "`validation_step` should not run when `limit_val_batches=0`"
+    assert not model.validation_epoch_end_invoked, "`validation_epoch_end` should not run when `limit_val_batches=0`"
+
+    # check that limit_xxx_batches will be reset when they are default value and overfit_batches > 0
+    model = CurrentModel(**hparams)
+    trainer_options.update(overfit_batches=2)
+    trainer_options.update(limit_train_batches=1.0)
+    trainer_options.update(limit_val_batches=1.0)
+    trainer = Trainer(**trainer_options)
+    trainer.fit(model)
+
+    assert trainer.state.finished, f"Training failed with {trainer.state}"
+    assert trainer.current_epoch == 1
+    assert trainer.limit_train_batches == 2
+    assert trainer.limit_val_batches == 2
+    assert model.validation_step_invoked, "did not run `validation_step` with `fast_dev_run=True`"
+    assert model.validation_epoch_end_invoked, "did not run `validation_epoch_end` with `fast_dev_run=True`"
+
+
 @mock.patch("torch.Tensor.backward")
 def test_nan_loss_detection(backward_mock, tmpdir):
     class CurrentModel(BoringModel):