Skip to content

Update backward hook for PrecisionPlugin #10008

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Oct 19, 2021
Merged

Update backward hook for PrecisionPlugin #10008

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
af2d044
add run_backward
awaelchli Oct 18, 2021
112c819
add types
awaelchli Oct 18, 2021
d65454b
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 19, 2021
29c8227
Update pytorch_lightning/plugins/precision/native_amp.py
awaelchli Oct 19, 2021
06c69d7
fix indentation
awaelchli Oct 19, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions pytorch_lightning/plugins/precision/deepspeed_precision.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,9 @@ def backward(self, model: "pl.LightningModule", closure_loss: Tensor, *args: Any
deepspeed_engine = model.trainer.model
deepspeed_engine.backward(closure_loss, *args, **kwargs)

def _run_backward(self, tensor: Tensor, model: Module, *args: Any, **kwargs: Any) -> None:
model.backward(tensor, *args, **kwargs)

def clip_gradients(
self,
optimizer: Optimizer,
Expand Down
6 changes: 6 additions & 0 deletions pytorch_lightning/plugins/precision/native_amp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
from typing import Any, Callable, Dict, Generator, Union

import torch
from torch import Tensor
from torch.nn import Module
from torch.optim import LBFGS, Optimizer

import pytorch_lightning as pl
Expand Down Expand Up @@ -68,6 +70,10 @@ def pre_backward(self, model: "pl.LightningModule", closure_loss: torch.Tensor)
closure_loss = self.scaler.scale(closure_loss)
return super().pre_backward(model, closure_loss)

def _run_backward(self, tensor: Tensor, model: Module, *args: Any, **kwargs: Any) -> None:
tensor = self.scaler.scale(tensor)
super()._run_backward(tensor, model, *args, **kwargs)

def pre_optimizer_step(
self,
model: "pl.LightningModule",
Expand Down
9 changes: 8 additions & 1 deletion pytorch_lightning/plugins/precision/precision_plugin.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ def backward(
if model is not None and isinstance(model, pl.LightningModule):
model.backward(closure_loss, optimizer, *args, **kwargs)
else:
closure_loss.backward(*args, **kwargs)
self._run_backward(closure_loss, *args, **kwargs)

def post_backward(self, model: "pl.LightningModule", closure_loss: Tensor) -> Tensor:
"""Run after precision plugin executes backward.
Expand All @@ -90,6 +90,13 @@ def post_backward(self, model: "pl.LightningModule", closure_loss: Tensor) -> Te
model.trainer.call_hook("on_after_backward")
return closure_loss

def _run_backward(self, tensor: Tensor, model: Module, *args: Any, **kwargs: Any) -> None:
"""Lightning-independent backward logic.

Currently only used by Lightning Lite. Subject to further refactors.
"""
tensor.backward(*args, **kwargs)

def pre_optimizer_step(
self,
model: "pl.LightningModule",
Expand Down