diff --git a/pytorch_lightning/plugins/training_type/deepspeed.py b/pytorch_lightning/plugins/training_type/deepspeed.py index 4e5d164a4fc0e..fdfb89a96b8be 100644 --- a/pytorch_lightning/plugins/training_type/deepspeed.py +++ b/pytorch_lightning/plugins/training_type/deepspeed.py @@ -627,9 +627,9 @@ def _auto_select_batch_size(self): return batch_size def _format_precision_config(self): - amp_type = self.lightning_module.trainer.accelerator_connector.amp_type - amp_level = self.lightning_module.trainer.accelerator_connector.amp_level - precision = self.lightning_module.trainer.accelerator_connector.precision + amp_type = self.lightning_module.trainer._accelerator_connector.amp_type + amp_level = self.lightning_module.trainer._accelerator_connector.amp_level + precision = self.lightning_module.trainer._accelerator_connector.precision if precision in (16, "mixed"): if "fp16" not in self.config and amp_type == AMPType.NATIVE: # FP16 is a DeepSpeed standalone AMP implementation diff --git a/pytorch_lightning/trainer/configuration_validator.py b/pytorch_lightning/trainer/configuration_validator.py index b59e7fd56f62f..9e3316e301740 100644 --- a/pytorch_lightning/trainer/configuration_validator.py +++ b/pytorch_lightning/trainer/configuration_validator.py @@ -196,7 +196,7 @@ def __verify_dp_batch_transfer_support(trainer: "pl.Trainer", model: "pl.Lightni # TODO: Remove this blocker once batch transfer to device is integrated in Lightning for DP mode. batch_transfer_hooks = ("on_before_batch_transfer", "transfer_batch_to_device", "on_after_batch_transfer") for hook in batch_transfer_hooks: - if trainer.accelerator_connector.use_dp and is_overridden(hook, model): + if trainer._accelerator_connector.use_dp and is_overridden(hook, model): raise MisconfigurationException(f"Overriding `{hook}` is not supported in DP mode.") diff --git a/pytorch_lightning/trainer/connectors/logger_connector/logger_connector.py b/pytorch_lightning/trainer/connectors/logger_connector/logger_connector.py index d96d24a016599..4119cbd6c78a5 100644 --- a/pytorch_lightning/trainer/connectors/logger_connector/logger_connector.py +++ b/pytorch_lightning/trainer/connectors/logger_connector/logger_connector.py @@ -244,7 +244,7 @@ def _log_gpus_metrics(self) -> None: self.trainer.lightning_module.log(key, mem, prog_bar=False, logger=True) else: gpu_id = int(key.split("/")[0].split(":")[1]) - if gpu_id in self.trainer.accelerator_connector.parallel_device_ids: + if gpu_id in self.trainer._accelerator_connector.parallel_device_ids: self.trainer.lightning_module.log( key, mem, prog_bar=False, logger=True, on_step=True, on_epoch=False ) diff --git a/pytorch_lightning/trainer/data_loading.py b/pytorch_lightning/trainer/data_loading.py index 52b1efa681665..24206b8af1fc1 100644 --- a/pytorch_lightning/trainer/data_loading.py +++ b/pytorch_lightning/trainer/data_loading.py @@ -70,7 +70,7 @@ def _worker_check(self, dataloader: DataLoader, name: str) -> None: if not isinstance(dataloader, DataLoader): return - using_spawn = self.accelerator_connector._distrib_type == DistributedType.DDP_SPAWN + using_spawn = self._accelerator_connector._distrib_type == DistributedType.DDP_SPAWN num_cpus = multiprocessing.cpu_count() # ddp_spawn + num_workers > 0 don't mix! tell the user @@ -120,8 +120,8 @@ def auto_add_worker_init_fn(self, dataloader: DataLoader) -> None: def _requires_distributed_sampler(self, dataloader) -> bool: return ( - self.accelerator_connector.replace_sampler_ddp - and self.accelerator_connector.is_distributed + self._accelerator_connector.replace_sampler_ddp + and self._accelerator_connector.is_distributed and not isinstance(dataloader.sampler, DistributedSampler) and not has_iterable_dataset(dataloader) ) @@ -147,7 +147,7 @@ def prepare_dataloader(self, dataloader: Any, shuffle: bool, mode: Optional[Runn _fault_tolerant_training() # injects components to track the state or self._requires_distributed_sampler(dataloader) # sets the distributed sampler or mode == RunningStage.PREDICTING # to track indices for the predictions - or self.accelerator_connector.use_ipu # IPUs use a custom `DataLoader` + or self._accelerator_connector.use_ipu # IPUs use a custom `DataLoader` ): sampler = self._resolve_sampler(dataloader, shuffle=shuffle, mode=mode) dataloader = self._update_dataloader(dataloader, sampler, mode=mode) diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index 05608436f1af3..5fa568df3f658 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -428,7 +428,7 @@ def __init__( self._data_connector = DataConnector(self, multiple_trainloader_mode) self.optimizer_connector = OptimizerConnector(self) - self.accelerator_connector = AcceleratorConnector( + self._accelerator_connector = AcceleratorConnector( num_processes, devices, tpu_cores, @@ -1508,7 +1508,7 @@ def _on_exception(self): @property def accelerator(self) -> Accelerator: - return self.accelerator_connector.accelerator + return self._accelerator_connector.accelerator @property def training_type_plugin(self) -> TrainingTypePlugin: @@ -1543,43 +1543,43 @@ def should_rank_save_checkpoint(self) -> bool: @property def _distrib_type(self) -> DistributedType: - return self.accelerator_connector._distrib_type + return self._accelerator_connector._distrib_type @property def _device_type(self) -> DeviceType: - return self.accelerator_connector._device_type + return self._accelerator_connector._device_type @property def num_nodes(self) -> int: - return self.accelerator_connector.num_nodes + return self._accelerator_connector.num_nodes @property def num_processes(self) -> int: - return self.accelerator_connector.num_processes + return self._accelerator_connector.num_processes @property def root_gpu(self) -> Optional[int]: - return self.accelerator_connector.root_gpu + return self._accelerator_connector.root_gpu @property def tpu_cores(self) -> int: - return self.accelerator_connector.tpu_cores + return self._accelerator_connector.tpu_cores @property def ipus(self) -> int: - return self.accelerator_connector.num_ipus + return self._accelerator_connector.num_ipus @property def num_gpus(self) -> int: - return self.accelerator_connector.num_gpus + return self._accelerator_connector.num_gpus @property def devices(self) -> Optional[Union[List[int], str, int]]: - return self.accelerator_connector.devices + return self._accelerator_connector.devices @property def data_parallel_device_ids(self) -> Optional[List[int]]: - return self.accelerator_connector.parallel_device_ids + return self._accelerator_connector.parallel_device_ids @property def lightning_module(self) -> "pl.LightningModule": @@ -1628,7 +1628,7 @@ def scaler(self): @property def gpus(self) -> Optional[Union[List[int], str, int]]: - return self.accelerator_connector.gpus + return self._accelerator_connector.gpus @property def model(self) -> torch.nn.Module: diff --git a/tests/accelerators/test_accelerator_connector.py b/tests/accelerators/test_accelerator_connector.py index 78189def69ad5..e6a18d538eb1d 100644 --- a/tests/accelerators/test_accelerator_connector.py +++ b/tests/accelerators/test_accelerator_connector.py @@ -100,7 +100,7 @@ def test_accelerator_choice_ddp_spawn(cuda_available_mock, device_count_mock): def test_accelerator_choice_ddp_slurm(setup_distributed_mock): class CB(Callback): def on_fit_start(self, trainer, pl_module): - assert trainer.accelerator_connector._is_slurm_managing_tasks + assert trainer._accelerator_connector._is_slurm_managing_tasks assert isinstance(trainer.accelerator, GPUAccelerator) assert isinstance(trainer.training_type_plugin, DDPPlugin) assert isinstance(trainer.training_type_plugin.cluster_environment, SLURMEnvironment) @@ -132,7 +132,7 @@ def on_fit_start(self, trainer, pl_module): def test_accelerator_choice_ddp2_slurm(device_count_mock, setup_distributed_mock): class CB(Callback): def on_fit_start(self, trainer, pl_module): - assert trainer.accelerator_connector._is_slurm_managing_tasks + assert trainer._accelerator_connector._is_slurm_managing_tasks assert isinstance(trainer.accelerator, GPUAccelerator) assert isinstance(trainer.training_type_plugin, DDP2Plugin) assert isinstance(trainer.training_type_plugin.cluster_environment, SLURMEnvironment) @@ -307,7 +307,7 @@ def on_fit_start(self, trainer, pl_module): def test_accelerator_choice_ddp_cpu_slurm(device_count_mock, setup_distributed_mock): class CB(Callback): def on_fit_start(self, trainer, pl_module): - assert trainer.accelerator_connector._is_slurm_managing_tasks + assert trainer._accelerator_connector._is_slurm_managing_tasks assert isinstance(trainer.accelerator, CPUAccelerator) assert isinstance(trainer.training_type_plugin, DDPPlugin) assert isinstance(trainer.training_type_plugin.cluster_environment, SLURMEnvironment) @@ -402,7 +402,7 @@ class TrainTypePlugin(SingleDevicePlugin): assert isinstance(trainer.accelerator, Accel) assert isinstance(trainer.training_type_plugin, TrainTypePlugin) assert isinstance(trainer.precision_plugin, Prec) - assert trainer.accelerator_connector.training_type_plugin is ttp + assert trainer._accelerator_connector.training_type_plugin is ttp class DistributedPlugin(DDPPlugin): pass @@ -413,7 +413,7 @@ class DistributedPlugin(DDPPlugin): assert isinstance(trainer.accelerator, Accel) assert isinstance(trainer.training_type_plugin, DistributedPlugin) assert isinstance(trainer.precision_plugin, Prec) - assert trainer.accelerator_connector.training_type_plugin is ttp + assert trainer._accelerator_connector.training_type_plugin is ttp @mock.patch.dict( @@ -756,7 +756,7 @@ def test_strategy_choice_ddp_spawn(cuda_available_mock, device_count_mock): def test_strategy_choice_ddp_slurm(setup_distributed_mock): class CB(Callback): def on_fit_start(self, trainer, pl_module): - assert trainer.accelerator_connector._is_slurm_managing_tasks + assert trainer._accelerator_connector._is_slurm_managing_tasks assert isinstance(trainer.accelerator, GPUAccelerator) assert isinstance(trainer.training_type_plugin, DDPPlugin) assert isinstance(trainer.training_type_plugin.cluster_environment, SLURMEnvironment) @@ -788,7 +788,7 @@ def on_fit_start(self, trainer, pl_module): def test_strategy_choice_ddp2_slurm(device_count_mock, setup_distributed_mock): class CB(Callback): def on_fit_start(self, trainer, pl_module): - assert trainer.accelerator_connector._is_slurm_managing_tasks + assert trainer._accelerator_connector._is_slurm_managing_tasks assert isinstance(trainer.accelerator, GPUAccelerator) assert isinstance(trainer.training_type_plugin, DDP2Plugin) assert isinstance(trainer.training_type_plugin.cluster_environment, SLURMEnvironment) @@ -963,7 +963,7 @@ def on_fit_start(self, trainer, pl_module): def test_strategy_choice_ddp_cpu_slurm(device_count_mock, setup_distributed_mock): class CB(Callback): def on_fit_start(self, trainer, pl_module): - assert trainer.accelerator_connector._is_slurm_managing_tasks + assert trainer._accelerator_connector._is_slurm_managing_tasks assert isinstance(trainer.accelerator, CPUAccelerator) assert isinstance(trainer.training_type_plugin, DDPPlugin) assert isinstance(trainer.training_type_plugin.cluster_environment, SLURMEnvironment) diff --git a/tests/deprecated_api/test_remove_1-6.py b/tests/deprecated_api/test_remove_1-6.py index fd0984d671b3e..74ab533d39efb 100644 --- a/tests/deprecated_api/test_remove_1-6.py +++ b/tests/deprecated_api/test_remove_1-6.py @@ -411,13 +411,13 @@ def test_v1_6_0_deprecated_accelerator_pass_through_functions(): def test_v1_6_0_configure_slurm_ddp(): trainer = Trainer() with pytest.deprecated_call(match=r"`AcceleratorConnector.configure_slurm_ddp\(\)` was deprecated in v1.5"): - trainer.accelerator_connector.configure_slurm_ddp() + trainer._accelerator_connector.configure_slurm_ddp() def test_v1_6_0_is_slurm_managing_tasks(): trainer = Trainer() with pytest.deprecated_call(match=r"`AcceleratorConnector.is_slurm_managing_tasks` was deprecated in v1.5"): - _ = trainer.accelerator_connector.is_slurm_managing_tasks + _ = trainer._accelerator_connector.is_slurm_managing_tasks with pytest.deprecated_call(match=r"`AcceleratorConnector.is_slurm_managing_tasks` was deprecated in v1.5"): - trainer.accelerator_connector.is_slurm_managing_tasks = False + trainer._accelerator_connector.is_slurm_managing_tasks = False diff --git a/tests/models/test_gpu.py b/tests/models/test_gpu.py index 8dc0de250dfae..69fedeaed2665 100644 --- a/tests/models/test_gpu.py +++ b/tests/models/test_gpu.py @@ -239,8 +239,8 @@ def test_torchelastic_gpu_parsing(mocked_device_count, gpus): """Ensure when using torchelastic and nproc_per_node is set to the default of 1 per GPU device That we omit sanitizing the gpus as only one of the GPUs is visible.""" trainer = Trainer(gpus=gpus) - assert isinstance(trainer.accelerator_connector.cluster_environment, TorchElasticEnvironment) - assert trainer.accelerator_connector.parallel_device_ids == device_parser.parse_gpu_ids(gpus) + assert isinstance(trainer._accelerator_connector.cluster_environment, TorchElasticEnvironment) + assert trainer._accelerator_connector.parallel_device_ids == device_parser.parse_gpu_ids(gpus) assert trainer.gpus == gpus diff --git a/tests/models/test_tpu.py b/tests/models/test_tpu.py index 6db8965c12d00..d8ceb4106fd07 100644 --- a/tests/models/test_tpu.py +++ b/tests/models/test_tpu.py @@ -251,7 +251,7 @@ def test_dataloaders_passed_to_fit(tmpdir): @RunIf(tpu=True) def test_tpu_id_to_be_as_expected(tpu_cores, expected_tpu_id): """Test if trainer.tpu_id is set as expected.""" - assert Trainer(tpu_cores=tpu_cores).accelerator_connector.tpu_id == expected_tpu_id + assert Trainer(tpu_cores=tpu_cores)._accelerator_connector.tpu_id == expected_tpu_id def test_tpu_misconfiguration(): @@ -315,7 +315,7 @@ def test_tpu_choice(tmpdir, tpu_cores, expected_tpu_id, error_expected): Trainer(default_root_dir=tmpdir, tpu_cores=tpu_cores) else: trainer = Trainer(default_root_dir=tmpdir, tpu_cores=tpu_cores) - assert trainer.accelerator_connector.tpu_id == expected_tpu_id + assert trainer._accelerator_connector.tpu_id == expected_tpu_id @pytest.mark.parametrize( diff --git a/tests/overrides/test_data_parallel.py b/tests/overrides/test_data_parallel.py index 46cdcc7cf7e23..62a497b3106d1 100644 --- a/tests/overrides/test_data_parallel.py +++ b/tests/overrides/test_data_parallel.py @@ -87,7 +87,7 @@ def training_step(self, batch, batch_idx): model = TestModel() trainer = MagicMock() trainer.state.stage = RunningStage.TRAINING - trainer.accelerator_connector._init_deterministic(False) + trainer._accelerator_connector._init_deterministic(False) model.trainer = trainer batch = torch.rand(2, 32).cuda() @@ -128,7 +128,7 @@ def training_step(self, batch, batch_idx): model = TestModel().to(device) trainer = MagicMock() trainer.state.stage = RunningStage.TRAINING - trainer.accelerator_connector._init_deterministic(False) + trainer._accelerator_connector._init_deterministic(False) model.trainer = trainer batch = torch.rand(2, 32).to(device) batch_idx = 0 diff --git a/tests/trainer/test_data_loading.py b/tests/trainer/test_data_loading.py index 8b60b8f5a6e97..723cff55c6860 100644 --- a/tests/trainer/test_data_loading.py +++ b/tests/trainer/test_data_loading.py @@ -137,7 +137,7 @@ def _get_warning_msg(): @pytest.mark.parametrize("num_workers", [0, 1]) def test_dataloader_warnings(tmpdir, num_workers): trainer = Trainer(default_root_dir=tmpdir, strategy="ddp_spawn", num_processes=2, fast_dev_run=4) - assert trainer.accelerator_connector._distrib_type == DistributedType.DDP_SPAWN + assert trainer._accelerator_connector._distrib_type == DistributedType.DDP_SPAWN trainer.fit(TestSpawnBoringModel(num_workers)) diff --git a/tests/trainer/test_trainer.py b/tests/trainer/test_trainer.py index 16629d9da52c6..d4d601b85bb8a 100644 --- a/tests/trainer/test_trainer.py +++ b/tests/trainer/test_trainer.py @@ -1352,7 +1352,7 @@ def write_on_batch_end(self, trainer, pl_module, prediction, batch_indices, *arg self.write_on_batch_end_called = True def write_on_epoch_end(self, trainer, pl_module, predictions, batch_indices): - expected = 1 if trainer.accelerator_connector.is_distributed else 2 + expected = 1 if trainer._accelerator_connector.is_distributed else 2 assert len(predictions) == 2 assert len(predictions[0]) == expected assert len(batch_indices) == 2 @@ -1360,7 +1360,7 @@ def write_on_epoch_end(self, trainer, pl_module, predictions, batch_indices): self.write_on_epoch_end_called = True def on_predict_epoch_end(self, trainer, pl_module, outputs): - if trainer.accelerator_connector.is_distributed: + if trainer._accelerator_connector.is_distributed: for idx in range(2): assert isinstance(trainer.predict_dataloaders[idx].batch_sampler.sampler, UnrepeatedDistributedSampler) assert isinstance(trainer.predict_dataloaders[idx].batch_sampler, IndexBatchSamplerWrapper) diff --git a/tests/utilities/test_cli.py b/tests/utilities/test_cli.py index 2ebbc2a5b183b..a5f7bf87a86ac 100644 --- a/tests/utilities/test_cli.py +++ b/tests/utilities/test_cli.py @@ -321,7 +321,7 @@ def test_lightning_cli_args_cluster_environments(tmpdir): class TestModel(BoringModel): def on_fit_start(self): # Ensure SLURMEnvironment is set, instead of default LightningEnvironment - assert isinstance(self.trainer.accelerator_connector._cluster_environment, SLURMEnvironment) + assert isinstance(self.trainer._accelerator_connector._cluster_environment, SLURMEnvironment) self.trainer.ran_asserts = True with mock.patch("sys.argv", ["any.py", "fit", f"--trainer.plugins={json.dumps(plugins)}"]):