Merge branch 'master' into accel_conn

Author: tchaton

.github/workflows/ci_test-conda.yml

@@ -31,6 +31,7 @@ jobs:
         python ./requirements/adjust_versions.py requirements/extra.txt
         python ./requirements/adjust_versions.py requirements/examples.txt
         pip install --requirement requirements/devel.txt --find-links https://download.pytorch.org/whl/nightly/torch_nightly.html
+        pip install pytest-random-order
         pip list
 
     - name: Pull checkpoints from S3
@@ -44,7 +45,7 @@ jobs:
     - name: Tests
       run: |
         # NOTE: run coverage on tests does not propagate failure status for Win, https://github.com/nedbat/coveragepy/issues/1003
-        coverage run --source pytorch_lightning -m pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-torch${{ matrix.pytorch-version }}.xml
+        coverage run --source pytorch_lightning -m pytest --random-order-seed=1 pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-torch${{ matrix.pytorch-version }}.xml
       shell: bash -l {0}
 
     - name: Upload pytest results

docs/source/common/trainer.rst

@@ -516,7 +516,9 @@ Example::
 checkpoint_callback
 ^^^^^^^^^^^^^^^^^^^
 
-Deprecated: This has been deprecated in v1.5 and will be removed in v1.7. Please use ``enable_checkpointing`` instead.
+.. warning:: `checkpoint_callback` has been deprecated in v1.5 and will be removed in v1.7.
+    To disable checkpointing, pass ``enable_checkpointing = False`` to the Trainer instead.
+
 
 default_root_dir
 ^^^^^^^^^^^^^^^^

docs/source/extensions/callbacks.rst

@@ -72,10 +72,10 @@ Examples
 --------
 You can do pretty much anything with callbacks.
 
-- `Add a MLP to fine-tune self-supervised networks <https://lightning-bolts.readthedocs.io/en/latest/self_supervised_callbacks.html#sslonlineevaluator>`_.
-- `Find how to modify an image input to trick the classification result <https://lightning-bolts.readthedocs.io/en/latest/vision_callbacks.html#confused-logit>`_.
-- `Interpolate the latent space of any variational model <https://lightning-bolts.readthedocs.io/en/latest/variational_callbacks.html#latent-dim-interpolator>`_.
-- `Log images to Tensorboard for any model <https://lightning-bolts.readthedocs.io/en/latest/vision_callbacks.html#tensorboard-image-generator>`_.
+- `Add a MLP to fine-tune self-supervised networks <https://lightning-bolts.readthedocs.io/en/latest/deprecated/callbacks/self_supervised.html#sslonlineevaluator>`_.
+- `Find how to modify an image input to trick the classification result <https://lightning-bolts.readthedocs.io/en/latest/deprecated/callbacks/vision.html#confused-logit>`_.
+- `Interpolate the latent space of any variational model <https://lightning-bolts.readthedocs.io/en/latest/deprecated/callbacks/variational.html#latent-dim-interpolator>`_.
+- `Log images to Tensorboard for any model <https://lightning-bolts.readthedocs.io/en/latest/deprecated/callbacks/vision.html#tensorboard-image-generator>`_.
 
 
 --------------

pytorch_lightning/__about__.py

@@ -1,7 +1,7 @@
 import time
 
 _this_year = time.strftime("%Y")
-__version__ = "1.5.0rc0"
+__version__ = "1.5.0rc1"
 __author__ = "William Falcon et al."
 __author_email__ = "[email protected]"
 __license__ = "Apache-2.0"

pytorch_lightning/loops/dataloader/evaluation_loop.py

@@ -101,7 +101,7 @@ def advance(self, *args: Any, **kwargs: Any) -> None:
 
         dataloader_idx: int = self.current_dataloader_idx
         dataloader = self.trainer.training_type_plugin.process_dataloader(self.current_dataloader)
-        dataloader = self.trainer.data_connector.get_profiled_dataloader(dataloader, dataloader_idx=dataloader_idx)
+        dataloader = self.trainer._data_connector.get_profiled_dataloader(dataloader, dataloader_idx=dataloader_idx)
         dl_max_batches = self._max_batches[dataloader_idx]
 
         dl_outputs = self.epoch_loop.run(dataloader, dataloader_idx, dl_max_batches, self.num_dataloaders)

pytorch_lightning/loops/epoch/evaluation_epoch_loop.py

@@ -107,7 +107,7 @@ def advance(
         if batch is None:
             raise StopIteration
 
-        if not self.trainer.data_connector.evaluation_data_fetcher.store_on_device:
+        if not self.trainer._data_connector.evaluation_data_fetcher.store_on_device:
             with self.trainer.profiler.profile("evaluation_batch_to_device"):
                 batch = self.trainer.accelerator.batch_to_device(batch, dataloader_idx=dataloader_idx)
 

pytorch_lightning/loops/epoch/training_epoch_loop.py

@@ -147,7 +147,7 @@ def advance(self, *args: Any, **kwargs: Any) -> None:
 
         batch_idx, (batch, self.batch_progress.is_last_batch) = next(self._dataloader_iter)
 
-        if not self.trainer.data_connector.train_data_fetcher.store_on_device:
+        if not self.trainer._data_connector.train_data_fetcher.store_on_device:
             with self.trainer.profiler.profile("training_batch_to_device"):
                 batch = self.trainer.accelerator.batch_to_device(batch)
 

pytorch_lightning/loops/fit_loop.py

@@ -212,7 +212,7 @@ def on_advance_start(self) -> None:
     def advance(self) -> None:
         """Runs one whole epoch."""
         dataloader = self.trainer.training_type_plugin.process_dataloader(self.trainer.train_dataloader)
-        data_fetcher = self.trainer.data_connector.get_profiled_dataloader(dataloader)
+        data_fetcher = self.trainer._data_connector.get_profiled_dataloader(dataloader)
 
         with self.trainer.profiler.profile("run_training_epoch"):
             self.epoch_loop.run(data_fetcher)

pytorch_lightning/plugins/training_type/deepspeed.py

@@ -623,7 +623,7 @@ def _auto_select_batch_size(self):
         # train_micro_batch_size_per_gpu is used for throughput logging purposes
         # by default we try to use the batch size of the loader
         batch_size = 1
-        train_dl_source = self.lightning_module.trainer.data_connector._train_dataloader_source
+        train_dl_source = self.lightning_module.trainer._data_connector._train_dataloader_source
         if train_dl_source.is_defined():
             train_dataloader = train_dl_source.dataloader()
             if hasattr(train_dataloader, "batch_sampler"):

pytorch_lightning/plugins/training_type/sharded.py

@@ -73,7 +73,6 @@ def _setup_models_and_optimizers(
 
         optimizers = self._wrap_optimizers(optimizers)
         model = ShardedDataParallel(models[0], sharded_optimizer=optimizers, **self._ddp_kwargs)
-        setattr(model, "require_backward_grad_sync", False)  # TODO: needed?
         return [model], optimizers
 
     def _reinit_optimizers_with_oss(self, optimizers: List[Union[Optimizer, LightningOptimizer]]) -> List["OSS"]:

pytorch_lightning/plugins/training_type/sharded_spawn.py

@@ -64,7 +64,6 @@ def _setup_models_and_optimizers(
 
         optimizers = self._wrap_optimizers(optimizers)
         model = ShardedDataParallel(models[0], sharded_optimizer=optimizers, **self._ddp_kwargs)
-        setattr(model, "require_backward_grad_sync", False)  # TODO: needed?
         return [model], optimizers
 
     def _reinit_optimizers_with_oss(self, optimizers: List[Optimizer]) -> List["OSS"]:

pytorch_lightning/plugins/training_type/tpu_spawn.py

@@ -95,7 +95,7 @@ def _validate_dataloader(dataloaders: Union[List[DataLoader], DataLoader]) -> No
     @staticmethod
     def _validate_patched_dataloaders(model: "pl.LightningModule") -> None:
         """Validate and fail fast if the dataloaders were passed directly to fit."""
-        connector: DataConnector = model.trainer.data_connector
+        connector: DataConnector = model.trainer._data_connector
         sources = (
             connector._train_dataloader_source,
             connector._val_dataloader_source,

pytorch_lightning/profiler/__init__.py

@@ -146,7 +146,11 @@ def custom_processing_step(self, data):
 The output below shows the profiling for the action ``training_step_and_backward``.
 The user can provide ``PyTorchProfiler(record_functions={...})`` to extend the scope of profiled functions.
 
-.. note:: When using the PyTorch Profiler, wall clock time will not not be representative of the true wall clock time. This is due to forcing profiled operations to be measured synchronously, when many CUDA ops happen asynchronously. It is recommended to use this Profiler to find bottlenecks/breakdowns, however for end to end wall clock time use the `SimpleProfiler`.   # noqa: E501
+.. note::
+    When using the PyTorch Profiler, wall clock time will not not be representative of the true wall clock time.
+    This is due to forcing profiled operations to be measured synchronously, when many CUDA ops happen asynchronously.
+    It is recommended to use this Profiler to find bottlenecks/breakdowns, however for end to end wall clock time use
+    the `SimpleProfiler`.
 
 .. code-block::
 

pytorch_lightning/trainer/configuration_validator.py

@@ -65,7 +65,7 @@ def __verify_train_loop_configuration(trainer: "pl.Trainer", model: "pl.Lightnin
     # -----------------------------------
     # verify model has a train dataloader
     # -----------------------------------
-    has_train_dataloader = trainer.data_connector._train_dataloader_source.is_defined()
+    has_train_dataloader = trainer._data_connector._train_dataloader_source.is_defined()
     if not has_train_dataloader:
         raise MisconfigurationException(
             "No `train_dataloader()` method defined. Lightning `Trainer` expects as minimum a"
@@ -176,7 +176,7 @@ def __verify_eval_loop_configuration(model: "pl.LightningModule", stage: str) ->
 
 
 def __verify_predict_loop_configuration(trainer: "pl.Trainer", model: "pl.LightningModule") -> None:
-    has_predict_dataloader = trainer.data_connector._predict_dataloader_source.is_defined()
+    has_predict_dataloader = trainer._data_connector._predict_dataloader_source.is_defined()
     if not has_predict_dataloader:
         raise MisconfigurationException("Dataloader not found for `Trainer.predict`")
     # ----------------------------------------------

pytorch_lightning/trainer/data_loading.py

@@ -343,7 +343,7 @@ def reset_train_dataloader(self, model: Optional["pl.LightningModule"] = None) -
             apply_to_collection(self.train_dataloader, DataLoader, self._add_sampler_metadata_collate)
 
         # wrap the sequence of train loaders to a CombinedLoader object for computing the num_training_batches
-        self.train_dataloader = CombinedLoader(self.train_dataloader, self.data_connector.multiple_trainloader_mode)
+        self.train_dataloader = CombinedLoader(self.train_dataloader, self._data_connector.multiple_trainloader_mode)
 
         self.num_training_batches = len(self.train_dataloader) if has_len(self.train_dataloader) else float("inf")
 
@@ -488,7 +488,7 @@ def reset_val_dataloader(self, model: Optional["pl.LightningModule"] = None) ->
         Args:
             model: The `LightningModule` if called outside of the trainer scope.
         """
-        source = self.data_connector._val_dataloader_source
+        source = self._data_connector._val_dataloader_source
         pl_module = self.lightning_module or model
         has_step = is_overridden("validation_step", pl_module)
         if source.is_defined() and has_step:
@@ -502,7 +502,7 @@ def reset_test_dataloader(self, model: Optional["pl.LightningModule"] = None) ->
         Args:
             model: The `LightningModule` if called outside of the trainer scope.
         """
-        source = self.data_connector._test_dataloader_source
+        source = self._data_connector._test_dataloader_source
         pl_module = self.lightning_module or model
         has_step = is_overridden("test_step", pl_module)
         if source.is_defined() and has_step:
@@ -516,7 +516,7 @@ def reset_predict_dataloader(self, model: Optional["pl.LightningModule"] = None)
         Args:
             model: The `LightningModule` if called outside of the trainer scope.
         """
-        source = self.data_connector._predict_dataloader_source
+        source = self._data_connector._predict_dataloader_source
         pl_module = self.lightning_module or model
         if source.is_defined():
             self.num_predict_batches, self.predict_dataloaders = self._reset_eval_dataloader(
@@ -545,7 +545,7 @@ def request_dataloader(
         Returns:
             The requested dataloader
         """
-        source = getattr(self.data_connector, f"_{stage.dataloader_prefix}_dataloader_source")
+        source = getattr(self._data_connector, f"_{stage.dataloader_prefix}_dataloader_source")
 
         hook = f"{stage.dataloader_prefix}_dataloader"
         self.call_hook("on_" + hook, pl_module=model)

pytorch_lightning/trainer/trainer.py

@@ -424,7 +424,7 @@ def __init__(
         gpu_ids, tpu_cores = self._parse_devices(gpus, auto_select_gpus, tpu_cores)
 
         # init connectors
-        self.data_connector = DataConnector(self, multiple_trainloader_mode)
+        self._data_connector = DataConnector(self, multiple_trainloader_mode)
         self.optimizer_connector = OptimizerConnector(self)
 
         self._accelerator_connector = AcceleratorConnector(
@@ -514,7 +514,7 @@ def __init__(
         self.optimizer_connector.on_trainer_init()
 
         # init data flags
-        self.data_connector.on_trainer_init(
+        self._data_connector.on_trainer_init(
             check_val_every_n_epoch,
             reload_dataloaders_every_n_epochs,
             reload_dataloaders_every_epoch,
@@ -663,7 +663,7 @@ def _fit_impl(
             )
 
         # links data to the trainer
-        self.data_connector.attach_data(
+        self._data_connector.attach_data(
             model, train_dataloaders=train_dataloaders, val_dataloaders=val_dataloaders, datamodule=datamodule
         )
 
@@ -747,7 +747,7 @@ def _validate_impl(
             )
 
         # links data to the trainer
-        self.data_connector.attach_data(model, val_dataloaders=dataloaders, datamodule=datamodule)
+        self._data_connector.attach_data(model, val_dataloaders=dataloaders, datamodule=datamodule)
 
         self.validated_ckpt_path = self.__set_ckpt_path(
             ckpt_path, model_provided=model_provided, model_connected=self.lightning_module is not None
@@ -837,7 +837,7 @@ def _test_impl(
             )
 
         # links data to the trainer
-        self.data_connector.attach_data(model, test_dataloaders=dataloaders, datamodule=datamodule)
+        self._data_connector.attach_data(model, test_dataloaders=dataloaders, datamodule=datamodule)
 
         self.tested_ckpt_path = self.__set_ckpt_path(
             ckpt_path, model_provided=model_provided, model_connected=self.lightning_module is not None
@@ -921,7 +921,7 @@ def _predict_impl(
             )
 
         # links data to the trainer
-        self.data_connector.attach_data(model, predict_dataloaders=dataloaders, datamodule=datamodule)
+        self._data_connector.attach_data(model, predict_dataloaders=dataloaders, datamodule=datamodule)
 
         self.predicted_ckpt_path = self.__set_ckpt_path(
             ckpt_path, model_provided=model_provided, model_connected=self.lightning_module is not None
@@ -985,7 +985,7 @@ def tune(
             )
 
         # links data to the trainer
-        self.data_connector.attach_data(
+        self._data_connector.attach_data(
             model, train_dataloaders=train_dataloaders, val_dataloaders=val_dataloaders, datamodule=datamodule
         )
 
@@ -1027,7 +1027,7 @@ def _run(self, model: "pl.LightningModule") -> Optional[Union[_EVALUATE_OUTPUT,
         self.training_type_plugin.connect(model)
 
         # hook
-        self.data_connector.prepare_data()
+        self._data_connector.prepare_data()
         self.callback_connector._attach_model_callbacks()
 
         if self._ckpt_path and not self.training_type_plugin.restore_checkpoint_after_pre_dispatch:
@@ -1171,7 +1171,7 @@ def _post_dispatch(self):
         # these `teardown` calls are here instead of in `_call_teardown_hook` since they are internal teardowns
         # which need to happen before.
         self.accelerator.teardown()
-        self.data_connector.teardown()
+        self._data_connector.teardown()
         self._active_loop.teardown()
         self.logger_connector.teardown()
 
@@ -1258,7 +1258,7 @@ def _run_predict(self) -> Optional[_PREDICT_OUTPUT]:
             return self.predict_loop.run()
 
     def _run_sanity_check(self, ref_model):
-        using_val_step = self.data_connector._val_dataloader_source.is_defined() and is_overridden(
+        using_val_step = self._data_connector._val_dataloader_source.is_defined() and is_overridden(
             "validation_step", ref_model
         )
         should_sanity_check = using_val_step and self.num_sanity_val_steps > 0 and self.limit_val_batches > 0

pytorch_lightning/tuner/batch_size_scaling.py

@@ -51,7 +51,7 @@ def scale_batch_size(
             " If this is not the intended behavior, please remove either one."
         )
 
-    if not trainer.data_connector._train_dataloader_source.is_module():
+    if not trainer._data_connector._train_dataloader_source.is_module():
         raise MisconfigurationException(
             "The batch scaling feature cannot be used with dataloaders passed directly to `.fit()`."
             " Please disable the feature or incorporate the dataloader into the model."

pytorch_lightning/utilities/imports.py

@@ -44,7 +44,7 @@ def _module_available(module_path: str) -> bool:
         return False
 
 
-def _compare_version(package: str, op: Callable, version: str, use_base_version: bool = True) -> bool:
+def _compare_version(package: str, op: Callable, version: str, use_base_version: bool = False) -> bool:
     """Compare package version with some requirements.
 
     >>> _compare_version("torch", operator.ge, "0.1")

tests/core/test_datamodules.py

@@ -51,7 +51,7 @@ def test_can_prepare_data(local_rank, node_rank):
     local_rank.return_value = 0
     assert trainer.local_rank == 0
 
-    trainer.data_connector.prepare_data()
+    trainer._data_connector.prepare_data()
     assert dm.random_full is not None
 
     # local rank = 1   (False)
@@ -60,7 +60,7 @@ def test_can_prepare_data(local_rank, node_rank):
     local_rank.return_value = 1
     assert trainer.local_rank == 1
 
-    trainer.data_connector.prepare_data()
+    trainer._data_connector.prepare_data()
     assert dm.random_full is None
 
     # prepare_data_per_node = False (prepare across all nodes)
@@ -71,7 +71,7 @@ def test_can_prepare_data(local_rank, node_rank):
     node_rank.return_value = 0
     local_rank.return_value = 0
 
-    trainer.data_connector.prepare_data()
+    trainer._data_connector.prepare_data()
     assert dm.random_full is not None
 
     # global rank = 1   (False)
@@ -80,13 +80,13 @@ def test_can_prepare_data(local_rank, node_rank):
     node_rank.return_value = 1
     local_rank.return_value = 0
 
-    trainer.data_connector.prepare_data()
+    trainer._data_connector.prepare_data()
     assert dm.random_full is None
 
     node_rank.return_value = 0
     local_rank.return_value = 1
 
-    trainer.data_connector.prepare_data()
+    trainer._data_connector.prepare_data()
     assert dm.random_full is None
 
     # 2 dm
@@ -100,13 +100,13 @@ def test_can_prepare_data(local_rank, node_rank):
         # has been called
         # False
         dm._has_prepared_data = True
-        trainer.data_connector.prepare_data()
+        trainer._data_connector.prepare_data()
         dm_mock.assert_not_called()
 
         # has not been called
         # True
         dm._has_prepared_data = False
-        trainer.data_connector.prepare_data()
+        trainer._data_connector.prepare_data()
         dm_mock.assert_called_once()
 
 
@@ -629,7 +629,7 @@ def test_inconsistent_prepare_data_per_node(tmpdir):
         trainer = Trainer(prepare_data_per_node=False)
         trainer.model = model
         trainer.datamodule = dm
-        trainer.data_connector.prepare_data()
+        trainer._data_connector.prepare_data()
 
 
 DATALOADER = DataLoader(RandomDataset(1, 32))

tests/plugins/test_tpu_spawn.py

@@ -65,7 +65,7 @@ def test_error_iterable_dataloaders_passed_to_fit(
     model = BoringModelNoDataloaders()
     model.trainer = trainer
 
-    trainer.data_connector.attach_dataloaders(
+    trainer._data_connector.attach_dataloaders(
         model,
         train_dataloaders=train_dataloaders,
         val_dataloaders=val_dataloaders,

tests/profiler/test_profiler.py

@@ -293,6 +293,7 @@ def test_pytorch_profiler_trainer_ddp(tmpdir, pytorch_profiler):
         assert any(f"{local_rank}-validation_step" in f for f in files)
 
 
+@RunIf(special=True)
 @pytest.mark.parametrize("fast_dev_run", [1, 2, 3, 4, 5])
 @pytest.mark.parametrize("boring_model_cls", [ManualOptimBoringModel, BoringModel])
 def test_pytorch_profiler_trainer_fit(fast_dev_run, boring_model_cls, tmpdir):