add exception

Author: rohitgr7

pytorch_lightning/core/lightning.py

@@ -1478,7 +1478,12 @@ def untoggle_optimizer(self, optimizer_idx: int):
         # save memory
         self._param_requires_grad_state = {}
 
-    def clip_gradients(self, optimizer: Optimizer, gradient_clip_val: Union[int, float], gradient_clip_algorithm: str):
+    def clip_gradients(
+        self,
+        optimizer: Optimizer,
+        gradient_clip_val: Optional[Union[int, float]] = None,
+        gradient_clip_algorithm: Optional[Union[str, GradClipAlgorithmType]] = None,
+    ):
         """Handles gradient clipping internally.
 
         Note:
@@ -1491,24 +1496,45 @@ def clip_gradients(self, optimizer: Optimizer, gradient_clip_val: Union[int, flo
             gradient_clip_algorithm: The gradient clipping algorithm to use. Pass ``gradient_clip_algorithm="value"``
                 to clip by value, and ``gradient_clip_algorithm="norm"`` to clip by norm.
         """
-        # gradient clipping
+        if gradient_clip_val is None:
+            gradient_clip_val = self.trainer.gradient_clip_val or 0.0
+        elif self.trainer.gradient_clip_val is not None and self.trainer.gradient_clip_val != gradient_clip_val:
+            raise MisconfigurationException(
+                "You have set `Trainer(gradient_clip_val)` and have passed"
+                " `gradient_clip_val` inside `clip_gradients`. Please use only one of them."
+            )
+
+        if gradient_clip_algorithm is None:
+            gradient_clip_algorithm = self.trainer.gradient_clip_algorithm or "norm"
+        else:
+            gradient_clip_algorithm = gradient_clip_algorithm.lower()
+            if (
+                self.trainer.gradient_clip_algorithm is not None
+                and self.trainer.gradient_clip_algorithm != gradient_clip_algorithm
+            ):
+                raise MisconfigurationException(
+                    "You have set `Trainer(gradient_clip_algorithm)` and have passed"
+                    " `gradient_clip_algorithm` inside `clip_gradients`. Please use only one of them."
+                )
+
         if not isinstance(gradient_clip_val, (int, float)):
             raise TypeError(f"`gradient_clip_val` should be an int or a float. Got {gradient_clip_val}.")
 
         if not GradClipAlgorithmType.supported_type(gradient_clip_algorithm.lower()):
             raise MisconfigurationException(
-                f"`gradient_clip_algorithm` {gradient_clip_algorithm} is invalid. "
-                f"Allowed algorithms: {GradClipAlgorithmType.supported_types()}."
+                f"`gradient_clip_algorithm` {gradient_clip_algorithm} is invalid."
+                f" Allowed algorithms: {GradClipAlgorithmType.supported_types()}."
             )
 
+        gradient_clip_algorithm = GradClipAlgorithmType(gradient_clip_algorithm)
         self.trainer.accelerator.clip_gradients(optimizer, gradient_clip_val, gradient_clip_algorithm)
 
     def configure_gradient_clipping(
         self,
         optimizer: Optimizer,
         optimizer_idx: int,
-        gradient_clip_val: Union[int, float],
-        gradient_clip_algorithm: str,
+        gradient_clip_val: Optional[Union[int, float]] = None,
+        gradient_clip_algorithm: Optional[str] = None,
     ):
         """Perform gradient clipping for the optimizer parameters. Called before :meth:`optimizer_step`.
 

pytorch_lightning/trainer/configuration_validator.py

@@ -184,7 +184,7 @@ def __verify_dp_batch_transfer_support(self, model: "pl.LightningModule") -> Non
     def __verify_manual_optimization_support(self, model: "pl.LightningModule") -> None:
         if model.automatic_optimization:
             return
-        if self.trainer.gradient_clip_val > 0:
+        if self.trainer.gradient_clip_val is not None and self.trainer.gradient_clip_val > 0:
             raise MisconfigurationException(
                 "Automatic gradient clipping is not supported for manual optimization."
                 f" Remove `Trainer(gradient_clip_val={self.trainer.gradient_clip_val})`"

pytorch_lightning/trainer/connectors/training_trick_connector.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Union
+from typing import Optional, Union
 
 from pytorch_lightning.utilities import GradClipAlgorithmType
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
@@ -23,19 +23,21 @@ def __init__(self, trainer):
 
     def on_trainer_init(
         self,
-        gradient_clip_val: Union[int, float],
-        gradient_clip_algorithm: str,
+        gradient_clip_val: Optional[Union[int, float]],
+        gradient_clip_algorithm: Optional[str],
         track_grad_norm: Union[int, float, str],
         terminate_on_nan: bool,
     ):
         if not isinstance(terminate_on_nan, bool):
             raise TypeError(f"`terminate_on_nan` should be a bool, got {terminate_on_nan}.")
 
         # gradient clipping
-        if not isinstance(gradient_clip_val, (int, float)):
+        if gradient_clip_val is not None and not isinstance(gradient_clip_val, (int, float)):
             raise TypeError(f"`gradient_clip_val` should be an int or a float. Got {gradient_clip_val}.")
 
-        if not GradClipAlgorithmType.supported_type(gradient_clip_algorithm.lower()):
+        if gradient_clip_algorithm is not None and not GradClipAlgorithmType.supported_type(
+            gradient_clip_algorithm.lower()
+        ):
             raise MisconfigurationException(
                 f"`gradient_clip_algorithm` {gradient_clip_algorithm} is invalid. "
                 f"Allowed algorithms: {GradClipAlgorithmType.supported_types()}."
@@ -49,5 +51,9 @@ def on_trainer_init(
 
         self.trainer.terminate_on_nan = terminate_on_nan
         self.trainer.gradient_clip_val = gradient_clip_val
-        self.trainer.gradient_clip_algorithm = GradClipAlgorithmType(gradient_clip_algorithm.lower())
+        self.trainer.gradient_clip_algorithm = (
+            GradClipAlgorithmType(gradient_clip_algorithm.lower())
+            if gradient_clip_algorithm is not None
+            else gradient_clip_algorithm
+        )
         self.trainer.track_grad_norm = float(track_grad_norm)

pytorch_lightning/trainer/trainer.py

@@ -123,8 +123,8 @@ def __init__(
         checkpoint_callback: bool = True,
         callbacks: Optional[Union[List[Callback], Callback]] = None,
         default_root_dir: Optional[str] = None,
-        gradient_clip_val: Union[int, float] = 0.0,
-        gradient_clip_algorithm: str = "norm",
+        gradient_clip_val: Optional[Union[int, float]] = None,
+        gradient_clip_algorithm: Optional[str] = None,
         process_position: int = 0,
         num_nodes: int = 1,
         num_processes: int = 1,
@@ -240,11 +240,12 @@ def __init__(
 
             gpus: Number of GPUs to train on (int) or which GPUs to train on (list or str) applied per node
 
-            gradient_clip_val: The value at which to clip gradients. Passing ``gradient_clip_val=0`` disables gradient
-                clipping.
+            gradient_clip_val: The value at which to clip gradients. Passing ``gradient_clip_val=None`` disables
+                gradient clipping.
 
             gradient_clip_algorithm: The gradient clipping algorithm to use. Pass ``gradient_clip_algorithm="value"``
-                to clip by value, and ``gradient_clip_algorithm="norm"`` to clip by norm.
+                to clip by value, and ``gradient_clip_algorithm="norm"`` to clip by norm. By default it will
+                be set to ``"norm"``.
 
             limit_train_batches: How much of training dataset to check (float = fraction, int = num_batches).
 

tests/core/test_lightning_module.py

@@ -22,6 +22,7 @@
 from pytorch_lightning import Trainer
 from pytorch_lightning.loggers import TensorBoardLogger
 from pytorch_lightning.utilities import _TORCH_SHARDED_TENSOR_AVAILABLE
+from pytorch_lightning.utilities.exceptions import MisconfigurationException
 from tests.helpers import BoringModel
 from tests.helpers.runif import RunIf
 
@@ -338,6 +339,8 @@ def test_sharded_tensor_state_dict(tmpdir, single_process_pg):
 
 
 def test_lightning_module_configure_gradient_clipping(tmpdir):
+    """Test custom gradient clipping inside `configure_gradient_clipping` hook."""
+
     class TestModel(BoringModel):
 
         has_validated_gradients = False
@@ -366,3 +369,40 @@ def on_before_optimizer_step(self, optimizer, optimizer_idx):
     )
     trainer.fit(model)
     assert model.has_validated_gradients
+
+
+def test_lightning_module_configure_gradient_clipping_different_argument_values(tmpdir):
+    """Test that setting gradient clipping arguments in `Trainer` and cusotmizing gradient clipping inside
+    `configure_gradient_clipping` with different values raises an exception."""
+
+    class TestModel(BoringModel):
+        custom_gradient_clip_val = 1e-2
+
+        def configure_gradient_clipping(self, optimizer, optimizer_idx, gradient_clip_val, gradient_clip_algorithm):
+            self.clip_gradients(optimizer, gradient_clip_val=self.custom_gradient_clip_val)
+
+    model = TestModel()
+    trainer = Trainer(
+        default_root_dir=tmpdir, max_epochs=1, limit_train_batches=2, limit_val_batches=0, gradient_clip_val=1e-4
+    )
+    with pytest.raises(MisconfigurationException, match=r".*have set `Trainer\(gradient_clip_val\)` and have passed.*"):
+        trainer.fit(model)
+
+    class TestModel(BoringModel):
+        custom_gradient_clip_algorithm = "value"
+
+        def configure_gradient_clipping(self, optimizer, optimizer_idx, gradient_clip_val, gradient_clip_algorithm):
+            self.clip_gradients(optimizer, gradient_clip_algorithm=self.custom_gradient_clip_algorithm)
+
+    model = TestModel()
+    trainer = Trainer(
+        default_root_dir=tmpdir,
+        max_epochs=1,
+        limit_train_batches=2,
+        limit_val_batches=0,
+        gradient_clip_algorithm="norm",
+    )
+    with pytest.raises(
+        MisconfigurationException, match=r".*have set `Trainer\(gradient_clip_algorithm\)` and have passed.*"
+    ):
+        trainer.fit(model)

tests/models/test_hooks.py

@@ -309,12 +309,12 @@ def _auto_train_batch(trainer, model, batches, device=torch.device("cpu"), curre
                     dict(
                         name="clip_gradients",
                         args=(ANY,),
-                        kwargs=dict(gradient_clip_val=0.0, gradient_clip_algorithm="norm"),
+                        kwargs=dict(gradient_clip_val=None, gradient_clip_algorithm=None),
                     ),
                     dict(
                         name="configure_gradient_clipping",
                         args=(ANY, 0),
-                        kwargs=dict(gradient_clip_val=0.0, gradient_clip_algorithm="norm"),
+                        kwargs=dict(gradient_clip_val=None, gradient_clip_algorithm=None),
                     ),
                     dict(
                         name="optimizer_step",