From 17d2ac56d32e8251735889b750f5cbfa74a13ca7 Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 24 Nov 2022 21:20:56 +0100 Subject: [PATCH 01/42] placeholder --- src/lightning_app/model2cloud/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/lightning_app/model2cloud/__init__.py diff --git a/src/lightning_app/model2cloud/__init__.py b/src/lightning_app/model2cloud/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d From 505f6b19b9327cffc3d34f13da9430db18bb0da2 Mon Sep 17 00:00:00 2001 From: Jirka Date: Tue, 13 Dec 2022 14:29:05 +0100 Subject: [PATCH 02/42] porting code --- .../model2cloud/authentication.py | 53 +++ src/lightning_app/model2cloud/cloud_api.py | 426 ++++++++++++++++++ src/lightning_app/model2cloud/save.py | 284 ++++++++++++ src/lightning_app/model2cloud/utils.py | 76 ++++ tests/tests_app/model2cloud/__init__.py | 0 tests/tests_app/model2cloud/constants.py | 4 + tests/tests_app/model2cloud/test_model.py | 122 +++++ .../model2cloud/test_requirements.py | 73 +++ .../tests_app/model2cloud/test_source_code.py | 168 +++++++ .../tests_app/model2cloud/test_versioning.py | 100 ++++ tests/tests_app/model2cloud/utils.py | 10 + 11 files changed, 1316 insertions(+) create mode 100644 src/lightning_app/model2cloud/authentication.py create mode 100644 src/lightning_app/model2cloud/cloud_api.py create mode 100644 src/lightning_app/model2cloud/save.py create mode 100644 src/lightning_app/model2cloud/utils.py create mode 100644 tests/tests_app/model2cloud/__init__.py create mode 100644 tests/tests_app/model2cloud/constants.py create mode 100644 tests/tests_app/model2cloud/test_model.py create mode 100644 tests/tests_app/model2cloud/test_requirements.py create mode 100644 tests/tests_app/model2cloud/test_source_code.py create mode 100644 tests/tests_app/model2cloud/test_versioning.py create mode 100644 tests/tests_app/model2cloud/utils.py diff --git a/src/lightning_app/model2cloud/authentication.py b/src/lightning_app/model2cloud/authentication.py new file mode 100644 index 0000000000000..5622283bc4050 --- /dev/null +++ b/src/lightning_app/model2cloud/authentication.py @@ -0,0 +1,53 @@ +import json +import webbrowser + +import requests +from requests.models import HTTPBasicAuth + +from lightning.app.utilities.network import LightningClient +from lightning_app.model2cloud.utils import LIGHTNING_CLOUD_URL + + +def get_user_details(): + def _get_user_details(): + client = LightningClient() + user_details = client.auth_service_get_user() + return (user_details.username, user_details.api_key) + + username, api_key = _get_user_details() + return username, api_key + + +def get_username_from_api_key(api_key: str): + response = requests.get( + url=f"{LIGHTNING_CLOUD_URL}/v1/auth/user", + auth=HTTPBasicAuth("lightning", api_key), + ) + assert response.status_code == 200, ( + "API_KEY provided is either invalid or wasn't found in the database." + " Please ensure that you passed the correct API_KEY." + ) + return json.loads(response.content)["username"] + + +def _check_browser_runnable(): + try: + webbrowser.get() + return True + except webbrowser.Error: + return False + + +def authenticate(inp_api_key: str = ""): + if not inp_api_key: + if not _check_browser_runnable(): + raise ValueError( + "Couldn't find a runnable browser in the current system/server." + " In order to run the commands on this system, we suggest passing the `api_key`" + " after logging into https://lightning.ai." + ) + username, api_key = get_user_details() + return username, api_key + + username = get_username_from_api_key(inp_api_key) + return username, inp_api_key diff --git a/src/lightning_app/model2cloud/cloud_api.py b/src/lightning_app/model2cloud/cloud_api.py new file mode 100644 index 0000000000000..74a2cd1f61434 --- /dev/null +++ b/src/lightning_app/model2cloud/cloud_api.py @@ -0,0 +1,426 @@ +import json +import logging +import os +import sys +import tempfile +from typing import List + +import requests +import torch + +from lightning_app.model2cloud.authentication import authenticate +from lightning_app.model2cloud.save import ( + _download_and_extract_data_to, + _save_checkpoint_from_path, + _save_meta_data, + _save_model, + _save_model_code, + _save_model_weights, + _save_requirements_file, + _submit_data_to_url, + _write_and_save_requirements, + get_linked_output_dir, +) +from lightning_app.model2cloud.utils import ( + get_model_data, + LIGHTNING_CLOUD_URL, + LIGHTNING_STORAGE_FILE, + split_name, + stage, +) + +if os.getenv("LIGHTNING_MODEL_STORE_TESTING", 0): + from tests.constants import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR +else: + from lightning_app.model2cloud.utils import LIGHTNING_STORAGE_DIR + +import lightning as L +import pytorch_lightning as PL + +logging.basicConfig(level=logging.INFO) + + +def to_lightning_cloud( + name: str, + version: str = "latest", + model=None, + source_code_path: str = "", + checkpoint_path: str = "", + requirements_file_path: str = "", + requirements: List[str] = [], + weights_only: bool = False, + api_key: str = "", + project_id: str = "", + progress_bar: bool = True, + save_code: bool = True, + *args, + **kwargs, +): + """ + Parameters + ========== + + :param: name: (str) + The model name. Model/Checkpoint will be uploaded with this unique name. Format: "model_name" + :param: version: (str, default="latest") + The version of the model to be uploaded. If not provided, default will be latest (not overridden). + :param: model: (default=None) + The model object (initialized). This is optional, but if `checkpoint_path` is not passed, + it will raise an error. (Optional) + :param: source_code_path: (str, default="") + The path to the source code that needs to be uploaded along with the model. + The path can point to a python file or a directory. Path pointing to a non-python file + will raise an error. (Optional) + :param: checkpoint_path (str, default="") + The path to the checkpoint that needs to be uploaded. (Optional) + :param: requirements_file_path (str, default="") + The path to the requirements file, will always be uploaded with the name of `requirements.txt`. + :param: requirements (List[str], default=[]) + List of requirements as strings, that will be written as `requirements.txt` and + then uploaded. If both `requirements_file_path` and `requirements` are passed, + a warning is raised as `requirements` is given the priority over `requirements_file_path`. + :param: weights_only (bool, default=False) + If set to `True`, it will only save model weights and nothing else. This raises + an error if `weights_only` is `True` but no `model` is passed. + :param: api_key (str, default="") + API_KEY used for authentication. Fetch it after logging to https://lightning.ai + (in the keys tab in the settings). If not passed, the API will attempt to + either find the credentials in your system or opening the login prompt. + :param: project_id (str, default="") + Some users have multiple projects with unique `project_id`. They need to pass + this in order to upload models to the cloud. + :param: progress_bar (bool, default=True) + A progress bar to show the uploading status. Disable this if not needed, by setting to `False`. + :param: save_code (bool, default=True) + By default, the API saves the code where the model is defined. + Set it to `False` if saving code is not desired. + + Returns + ======== + None + """ + if model is None and checkpoint_path is None: + raise ValueError( + """" + You either need to pass the model or the checkpoint path that you want to save. :) + Any one of: `to_lightning_cloud("model_name", model=modelObj, ...)` + or `to_lightning_cloud("model_name", checkpoint_path="your_checkpoint_path.ckpt", ...)` + is required. + """ + ) + + if weights_only and not model: + raise ValueError( + "No model passed to `to_lightning_cloud(...), in order to save weights," + " you need to pass the model object." + ) + + version = version or "latest" + _, model_name, _ = split_name(name, version=version, l_stage=stage.UPLOAD) + username_from_api_key, api_key = authenticate(api_key) + + name = f"{username_from_api_key}/{model_name}:{version}" + + stored = {} + + with tempfile.TemporaryDirectory() as tmpdir: + if checkpoint_path: + stored = _save_checkpoint_from_path( + model_name, + path=checkpoint_path, + tmpdir=tmpdir, + stored=stored, + ) + + if model: + stored = _save_model_weights( + model_name, + model_state_dict=model.state_dict(), + tmpdir=tmpdir, + stored=stored, + *args, + **kwargs, + ) + + if not weights_only: + stored = _save_model( + model_name, + model=model, + tmpdir=tmpdir, + stored=stored, + *args, + **kwargs, + ) + + if save_code: + stored = _save_model_code( + model_name, + model_cls=model.__class__, + source_code_path=source_code_path, + tmpdir=tmpdir, + stored=stored, + ) + + if requirements and requirements_file_path: + # TODO: Later on, figure out how to merge requirements from both args + logging.warning( + "You provided a requirements file (requirements_file_path=...)" + " and requirements list (requirements=...). In case of any collisions," + " anything that comes from requirements=... will be given the priority." + ) + + if requirements: + stored = _write_and_save_requirements( + model_name, + requirements=requirements, + stored=stored, + tmpdir=tmpdir, + ) + elif requirements_file_path: + stored = _save_requirements_file( + model_name, + requirements_file_path=requirements_file_path, + stored=stored, + tmpdir=tmpdir, + ) + + url = _save_meta_data( + model_name, + stored=stored, + version=version, + model=model, + username=username_from_api_key, + api_key=api_key, + project_id=project_id, + ) + + _submit_data_to_url(url, tmpdir, progress_bar=progress_bar) + msg = "Finished storing the following data items to the Lightning Cloud.\n" + for key, val in stored.items(): + if key == "code": + if val["type"] == "file": + msg += f"Stored code as a file with name: {val['path']}\n" + else: + msg += f"Stored code as a folder with name: {val['path']}\n" + else: + msg += f"Stored {key} with name: {val}\n" + + msg += ( + f'\nJust do: download_from_lightning_cloud("{username_from_api_key}/{model_name}", ' + f'version="{version}") in order to download the model from the cloud to your local system.' + ) + msg += ( + f'\nAnd: to_lightning_cloud("{username_from_api_key}/{model_name}", ' + f'version="{version}") in order to load the downloaded model.' + ) + logging.info(msg) + + +def _load_model(stored, output_dir, *args, **kwargs): + if "model" in stored: + sys.path.insert(0, f"{output_dir}") + model = torch.load(f"{output_dir}/{stored['model']}", *args, **kwargs) + return model + else: + raise ValueError( + "Couldn't find the model when uploaded to our storage. Please check" + " with the model owner to confirm that the models exist in the storage." + ) + + +def _load_weights(model, stored, output_dir, *args, **kwargs): + if "weights" in stored: + model.load_state_dict(torch.load(f"{output_dir}/{stored['weights']}", *args, **kwargs)) + return model + else: + raise ValueError( + "Weights were not found, please contact the model owner to verify if the" + " weights were stored successfully..." + ) + + +def _load_checkpoint(model, stored, output_dir, *args, **kwargs): + if "checkpoint" in stored: + ckpt = f"{output_dir}/{stored['checkpoint']}" + else: + raise ValueError( + "No checkpoint path was found, please contact the model owner to verify if the" + " checkpoint was saved successfully." + ) + + ckpt = model.load_from_checkpoint(ckpt, *args, **kwargs) + return ckpt + + +def download_from_lightning_cloud( + name: str, + version: str = "latest", + output_dir: str = "", + progress_bar: bool = True, +): + """ + Parameters + ========== + :param: name (str): + The unique name of the model to be downloaded. Format: `/`. + :param: version: (str, default="latest") + The version of the model to be uploaded. If not provided, default will be latest (not overridden). + :param: output_dir (str, default=""): + The target directory, where the model and other data will be stored. If not passed, + the data will be stored in `$HOME/.lightning/lightning_model_store///`. + (`version` defaults to `latest`) + + Returns + ======= + None + """ + version = version or "latest" + username, model_name, version = split_name(name, version=version, l_stage=stage.DOWNLOAD) + + linked_output_dir = "" + if not output_dir: + output_dir = LIGHTNING_STORAGE_DIR + output_dir = os.path.join(output_dir, username, model_name, version) + linked_output_dir = get_linked_output_dir(output_dir) + else: + output_dir = os.path.abspath(output_dir) + + if not os.path.isdir(output_dir): + os.makedirs(output_dir) + + response = requests.get(f"{LIGHTNING_CLOUD_URL}/v1/models?name={username}/{model_name}&version={version}") + assert response.status_code == 200, ( + f"Unable to download the model with name {name} and version {version}." + " Maybe reach out to the model owner or check the arguments again?" + ) + + download_url_response = json.loads(response.content) + download_url = download_url_response["downloadUrl"] + meta_data = download_url_response["metadata"] + + logging.info(f"Downloading the model data for {name} to {output_dir} folder.") + _download_and_extract_data_to(output_dir, download_url, progress_bar) + + if linked_output_dir: + logging.info(f"Linking the downloaded folder from {output_dir} to {linked_output_dir} folder.") + if os.path.islink(linked_output_dir): + os.unlink(linked_output_dir) + if os.path.exists(linked_output_dir): + if os.path.isdir(linked_output_dir): + os.rmdir(linked_output_dir) + + os.symlink(output_dir, linked_output_dir) + + with open(LIGHTNING_STORAGE_FILE, "w+") as storage_file: + storage = { + username: { + model_name: { + version: { + "output_dir": output_dir, + "linked_output_dir": str(linked_output_dir), + "metadata": meta_data, + }, + }, + }, + } + json.dump(storage, storage_file) + + logging.info("Downloading done...") + logging.info( + f"The source code for your model has been written to {output_dir} folder, and" + f" linked to {linked_output_dir} folder." + ) + + logging.info( + "Please make sure to add imports to the necessary classes needed for instantiation of" + " your model before calling `load_from_lightning_cloud`." + ) + + +def _validate_output_dir(dir: str): + if not os.path.exists(dir): + raise ValueError( + "The output directory doesn't exist... did you forget to call download_from_lightning_cloud(...)?" + ) + + +def load_from_lightning_cloud( + name: str, + version: str = "latest", + load_weights: bool = False, + load_checkpoint: bool = False, + model=None, + *args, + **kwargs, +): + """ + Parameters + ========== + :param: name (str) + Name of the model to load. Format: `/` + :param: version: (str, default="latest") + The version of the model to be uploaded. If not provided, default will be latest (not overridden). + :param: load_weights (bool, default=False) + Loads only weights if this is set to `True`. Needs `model` to be passed in order to load the weights. + :param: load_checkpoint (bool, default=False) + Loads checkpoint if this is set to `True`. Only a `LightningModule` model is supported for this feature. + + Returns + ======= + None + """ + if load_weights and load_checkpoint: + raise ValueError( + f"You passed load_weights={load_weights} and load_checkpoint={load_checkpoint}," + " it's expected that only one of them are requested in a single call." + ) + + if os.path.exists(LIGHTNING_STORAGE_FILE): + version = version or "latest" + model_data = get_model_data(name, version) + output_dir = model_data["output_dir"] + linked_output_dir = model_data["linked_output_dir"] + meta_data = model_data["metadata"] + stored = {"code": {}} + + for key, val in meta_data.items(): + if key.startswith("stored_"): + if key.startswith("stored_code_"): + stored["code"][key.split("_code_")[1]] = val + else: + stored[key.split("_")[1]] = val + + _validate_output_dir(output_dir) + if linked_output_dir: + _validate_output_dir(linked_output_dir) + + if load_weights: + # This first loads the model - and then the weights + if not model: + raise ValueError( + "Expected model=... to be passed for loading weights, please pass" + f" your model object to load_from_lightning_cloud({name}, {version}, model=ModelObj)" + ) + return _load_weights(model, stored, linked_output_dir or output_dir, *args, **kwargs) + elif load_checkpoint: + if not model: + raise ValueError( + "You need to pass the LightningModule object (model) to be able to" + f" load the checkpoint. `load_from_lightning_cloud({name}, {version}," + " load_checkpoint=True, model=...)`" + ) + if not isinstance(model, (PL.LightningModule, L.LightningModule)): + raise TypeError( + "For loading checkpoints, the model is required to be a LightningModule" + f" or a subclass of LightningModule, got type {type(model)}." + ) + + return _load_checkpoint(model, stored, linked_output_dir or output_dir, *args, **kwargs) + else: + return _load_model(stored, linked_output_dir or output_dir, *args, **kwargs) + else: + raise ValueError( + f"Could not find the model (for {name}:{version}) in the local system." + " Did you make sure to download the model using: `download_from_lightning_cloud(...)`" + " before calling `load_from_lightning_cloud(...)`?" + ) diff --git a/src/lightning_app/model2cloud/save.py b/src/lightning_app/model2cloud/save.py new file mode 100644 index 0000000000000..9d4ffb28a08ff --- /dev/null +++ b/src/lightning_app/model2cloud/save.py @@ -0,0 +1,284 @@ +import inspect +import json +import logging +import os +import shutil +import tarfile +from pathlib import PurePath + +import requests +import torch +from requests.auth import HTTPBasicAuth +from tqdm import tqdm +from tqdm.utils import CallbackIOWrapper + +from lightning_app.model2cloud.utils import LIGHTNING_CLOUD_URL + +logging.basicConfig(level=logging.INFO) + + +def _check_id(id: str): + if id[-1] != "/": + id += "/" + assert id.count("/") == 2, "The format for the ID should be: /" + return id + + +def _save_checkpoint(name, checkpoint, tmpdir, stored): + checkpoint_file_path = f"{tmpdir}/checkpoint.ckpt" + torch.save(checkpoint, checkpoint_file_path) + stored["checkpoint"] = "checkpoint.ckpt" + return stored + + +def _save_checkpoint_from_path(name, path, tmpdir, stored): + checkpoint_file_path = path + shutil.copy(checkpoint_file_path, f"{tmpdir}/checkpoint.ckpt") + stored["checkpoint"] = "checkpoint.ckpt" + return stored + + +def _save_model_weights(name, model_state_dict, tmpdir, stored, *args, **kwargs): + # For now we assume that it's always going to be public + weights_file_path = f"{tmpdir}/weights.pt" + torch.save(model_state_dict, weights_file_path, *args, **kwargs) + stored["weights"] = "weights.pt" + return stored + + +def _save_model(name, model, tmpdir, stored, *args, **kwargs): + # For now we assume that it's always going to be public + model_file_path = f"{tmpdir}/model" + torch.save(model, model_file_path, *args, **kwargs) + stored["model"] = "model" + return stored + + +def _save_model_code(name, model_cls, source_code_path, tmpdir, stored): + if source_code_path: + source_code_path = os.path.abspath(source_code_path) + assert os.path.exists(source_code_path), f"Given path {source_code_path} does not exist." + + # Copy contents to tmpdir folder + if os.path.isdir(source_code_path): + logging.warning( + f"NOTE: Folder: {source_code_path} is being uploaded to the cloud so that the user " + " can make the necessary imports after downloading your model." + ) + dir_name = os.path.basename(source_code_path) + shutil.copytree(source_code_path, f"{tmpdir}/{dir_name}/") + stored["code"] = {"type": "folder", "path": f"{dir_name}"} + else: + assert os.path.splitext(source_code_path)[-1] == ".py", ( + "Expected a Python file or a directory, to be uploaded for model definition," + f" but found {source_code_path}. If your file is not a Python file, and you still" + " want to save it, please consider saving it in a folder and passing the folder" + " path instead." + ) + + logging.warning( + f"NOTE: File: {source_code_path} is being uploaded to the cloud so that the" + " user can make the necessary imports after downloading your model." + ) + + file_name = os.path.basename(source_code_path) + shutil.copy(source_code_path, f"{tmpdir}/{file_name}") + stored["code"] = {"type": "file", "path": f"{file_name}"} + else: + # TODO: Raise a warning if the file has any statements/expressions outside of + # __name__ == "__main__" in the script + # As those will be executed on import + model_class_path = inspect.getsourcefile(model_cls) + if model_class_path: + assert os.path.splitext(model_class_path)[-1] == ".py", ( + f"The model definition was found in a non-python file ({model_class_path})," + " which is not currently supported (for safety reasons). If your file is not a" + " Python file, and you still want to save it, please consider saving it in a" + " folder and passing the folder path instead." + ) + + file_name = os.path.basename(model_class_path) + logging.warning( + f"NOTE: File: {model_class_path} is being uploaded to the cloud so that the" + " user can make the necessary imports after downloading your model. The file" + f" will be saved as {file_name} on download." + ) + shutil.copyfile(model_class_path, f"{tmpdir}/{file_name}") + stored["code"] = {"type": "file", "path": file_name} + return stored + + +def _write_and_save_requirements(name, requirements, stored, tmpdir): + if not isinstance(requirements, list) and isinstance(requirements, str): + requirements = [requirements] + + requirements_file_path = f"{tmpdir}/requirements.txt" + + with open(requirements_file_path, "w+") as req_file: + for req in requirements: + req_file.write(req + "\n") + + stored["requirements"] = "requirements.txt" + return stored + + +def _save_requirements_file(name, requirements_file_path, stored, tmpdir): + shutil.copyfile(os.path.abspath(requirements_file_path), f"{tmpdir}/requirements.txt") + stored["requirements"] = requirements_file_path + return stored + + +def _upload_metadata( + meta_data: dict, + name: str, + version: str, + username: str, + api_key: str, + project_id: str, +): + def _get_url(response_content): + content = json.loads(response_content) + return content["uploadUrl"] + + json_field = { + "name": f"{username}/{name}", + "version": version, + "metadata": meta_data, + } + if project_id: + json_field["project_id"] = project_id + response = requests.post( + f"{LIGHTNING_CLOUD_URL}/v1/models", + auth=HTTPBasicAuth(username, api_key), + json=json_field, + ) + + assert ( + response.status_code == 200 + ), f"Unable to upload content, did you pass correct credentials? Error: {response.content}" + return _get_url(response.content) + + +def _save_meta_data(name, stored, version, model, username, api_key, project_id): + def _process_stored(stored: dict): + processed_dict = {} + for key, val in stored.items(): + if "code" in key: + for code_key, code_val in stored[key].items(): + processed_dict[f"stored_code_{code_key}"] = code_val + else: + processed_dict[f"stored_{key}"] = val + return processed_dict + + meta_data = { + "cls": model.__class__.__name__, + } + + meta_data.update(_process_stored(stored)) + + return _upload_metadata( + meta_data, + name=name, + version=version, + username=username, + api_key=api_key, + project_id=project_id, + ) + + +def _submit_data_to_url(url: str, tmpdir: str, progress_bar: bool): + def _make_tar(tmpdir, archive_output_path): + with tarfile.open(archive_output_path, "w:gz") as tar: + tar.add(tmpdir) + + def upload_from_file(src, dst): + file_size = os.path.getsize(src) + with open(src, "rb") as fd: + with tqdm( + desc="Uploading", + total=file_size, + unit="B", + unit_scale=True, + unit_divisor=1024, + ) as t: + reader_wrapper = CallbackIOWrapper(t.update, fd, "read") + response = requests.put(dst, data=reader_wrapper) + response.raise_for_status() + + archive_path = f"{tmpdir}/data.tar.gz" + _make_tar(tmpdir, archive_path) + if progress_bar: + upload_from_file(archive_path, url) + else: + requests.put(url, data=open(archive_path, "rb")) + + +def _download_and_extract_data_to(output_dir: str, download_url: str, progress_bar: bool): + def _common_clean_up(): + data_file_path = f"{output_dir}/data.tar.gz" + dir_file_path = f"{output_dir}/extracted" + if os.path.exists(data_file_path): + os.remove(data_file_path) + shutil.rmtree(dir_file_path) + + try: + with requests.get(download_url, stream=True) as req_stream: + total_size_in_bytes = int(req_stream.headers.get("content-length", 0)) + block_size = 1024 # 1 Kibibyte + + download_progress_bar = None + if progress_bar: + download_progress_bar = tqdm(total=total_size_in_bytes, unit="iB", unit_scale=True) + with open(f"{output_dir}/data.tar.gz", "wb") as f: + for chunk in req_stream.iter_content(chunk_size=block_size): + if download_progress_bar: + download_progress_bar.update(len(chunk)) + f.write(chunk) + if download_progress_bar: + download_progress_bar.close() + + tar = tarfile.open(f"{output_dir}/data.tar.gz", "r:gz") + tmpdir_name = tar.getnames()[0] + tar.extractall(path=f"{output_dir}/extracted") + tar.close() + + root = f"{output_dir}" + for filename in os.listdir(os.path.join(root, "extracted", tmpdir_name)): + abs_file_name = os.path.join(root, "extracted", tmpdir_name, filename) + if os.path.isdir(abs_file_name): + func = shutil.copytree + else: + func = shutil.copy + + dst_file_name = os.path.join(root, filename) + if os.path.exists(dst_file_name): + if os.path.isdir(dst_file_name): + shutil.rmtree(dst_file_name) + else: + os.remove(dst_file_name) + + func( + abs_file_name, + os.path.join(root, filename), + ) + + assert os.path.isdir(f"{output_dir}"), ( + "Data downloading to the output" + f" directory: {output_dir} failed. Maybe try again or contact the model owner?" + ) + except Exception as e: + _common_clean_up() + raise e + else: + _common_clean_up() + + +def get_linked_output_dir(src_dir: str): + # The last sub-folder will be our version + version_folder_name = PurePath(src_dir).parts[-1] + + if version_folder_name == "latest": + return str(PurePath(src_dir).parent.joinpath("version_latest")) + else: + replaced_ver = version_folder_name.replace(".", "_") + return str(PurePath(src_dir).parent.joinpath(f"version_{replaced_ver}")) diff --git a/src/lightning_app/model2cloud/utils.py b/src/lightning_app/model2cloud/utils.py new file mode 100644 index 0000000000000..03bef838582b0 --- /dev/null +++ b/src/lightning_app/model2cloud/utils.py @@ -0,0 +1,76 @@ +import json +import os +from enum import Enum + + +class stage(Enum): + UPLOAD = 0 + LOAD = 1 + DOWNLOAD = 2 + + +LIGHTNING_DIR = f"{os.path.expanduser('~')}/.lightning" +LIGHTNING_STORAGE_FILE = f"{LIGHTNING_DIR}/.lightning_model_storage" +LIGHTNING_STORAGE_DIR = f"{LIGHTNING_DIR}/lightning_model_store" +LIGHTNING_CLOUD_URL = os.getenv("LIGHTNING_CLOUD_URL", default="https://lightning.ai") + + +def _check_version(version: str): + allowed_chars = "0123456789." + if version == "latest": + return True + for version_char in version: + if version_char not in allowed_chars: + return False + return True + + +def _split_name(name: str, version: str, l_stage: stage): + if l_stage == stage.UPLOAD: + username = "" + model_name = name + else: + username, model_name = name.split("/") + + return username, model_name, version + + +def split_name(name: str, version: str, l_stage: stage): + username, model_name, version = _split_name(name, version, l_stage) + + return ( + username, + model_name, + version, + ) + + +def get_model_data(name: str, version: str): + username, model_name, version = split_name(name, version, stage.LOAD) + + assert os.path.exists( + LIGHTNING_STORAGE_FILE + ), f"ERROR: Could not find {LIGHTNING_STORAGE_FILE} (to be generated after download_from_lightning_cloud(...))" + + with open(LIGHTNING_STORAGE_FILE) as storage_file: + storage_data = json.load(storage_file) + + assert username in storage_data, ( + f"No data found for the given username {username}. Make sure to call" + " `download_from_lightning_cloud` before loading" + ) + user_data = storage_data[username] + + assert model_name in user_data, ( + f"No data found for the given model name: {model_name} for the given" + f" username: {username}. Make sure to call `download_from_lightning_cloud` before loading" + ) + model_data = user_data[model_name] + + version = version or "latest" + assert ( + version in model_data + ), f"No data found for the given version: {version}, did you download the model successfully?" + model_version_data = model_data[version] + + return model_version_data diff --git a/tests/tests_app/model2cloud/__init__.py b/tests/tests_app/model2cloud/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/tests_app/model2cloud/constants.py b/tests/tests_app/model2cloud/constants.py new file mode 100644 index 0000000000000..be8e23f466544 --- /dev/null +++ b/tests/tests_app/model2cloud/constants.py @@ -0,0 +1,4 @@ +import os + +LIGHTNING_DIR = f"{os.path.expanduser('~')}/.lightning" +LIGHTNING_TEST_STORAGE_DIR = f"{LIGHTNING_DIR}/lightning_test_model_store/" diff --git a/tests/tests_app/model2cloud/test_model.py b/tests/tests_app/model2cloud/test_model.py new file mode 100644 index 0000000000000..b1c78aeeb472d --- /dev/null +++ b/tests/tests_app/model2cloud/test_model.py @@ -0,0 +1,122 @@ +import os + +from models_cloud import download_from_lightning_cloud, load_from_lightning_cloud, to_lightning_cloud +from tests.utils import cleanup + +import pytorch_lightning as pl +from pytorch_lightning.demos.boring_classes import BoringModel + +if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): + from tests_app.model2cloud.constants import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR +else: + from lightning_app.model2cloud.utils import LIGHTNING_STORAGE_DIR + + +def test_model(): + cleanup() + username = os.getenv("API_USERNAME", "") + if not username: + raise ValueError( + "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" + ) + + model_name = "boring_model" + version = "latest" + + to_lightning_cloud( + model_name, + model=BoringModel(), + api_key=os.getenv("API_KEY", ""), + project_id=os.getenv("PROJECT_ID", ""), + ) + + download_from_lightning_cloud(f"{username}/{model_name}") + assert os.path.isdir(os.path.join(LIGHTNING_STORAGE_DIR, username, model_name, version)) + + model = load_from_lightning_cloud(f"{username}/{model_name}") + assert model is not None + + +def test_model_without_progress_bar(): + cleanup() + username = os.getenv("API_USERNAME", "") + if not username: + raise ValueError( + "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" + ) + + model_name = "boring_model" + version = "latest" + + to_lightning_cloud( + model_name, + model=BoringModel(), + api_key=os.getenv("API_KEY", ""), + project_id=os.getenv("PROJECT_ID", ""), + progress_bar=False, + ) + + download_from_lightning_cloud(f"{username}/{model_name}", progress_bar=False) + assert os.path.isdir(os.path.join(LIGHTNING_STORAGE_DIR, username, model_name, version)) + + model = load_from_lightning_cloud(f"{username}/{model_name}") + assert model is not None + + +def test_only_weights(): + cleanup() + username = os.getenv("API_USERNAME", "") + if not username: + raise ValueError( + "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" + ) + + model_name = "boring_model_only_weights" + version = "latest" + + model = BoringModel() + trainer = pl.Trainer(fast_dev_run=True) + trainer.fit(model) + to_lightning_cloud( + model_name, + model=model, + weights_only=True, + api_key=os.getenv("API_KEY", ""), + project_id=os.getenv("PROJECT_ID", ""), + ) + + download_from_lightning_cloud(f"{username}/{model_name}") + assert os.path.isdir(os.path.join(LIGHTNING_STORAGE_DIR, username, model_name, version)) + + model_with_weights = load_from_lightning_cloud(f"{username}/{model_name}", load_weights=True, model=model) + assert model_with_weights is not None + assert model_with_weights.state_dict() is not None + + +def test_checkpoint_path(): + cleanup() + username = os.getenv("API_USERNAME", "") + if not username: + raise ValueError( + "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" + ) + + model_name = "boring_model_only_checkpoint_path" + version = "latest" + + model = BoringModel() + trainer = pl.Trainer(fast_dev_run=True) + trainer.fit(model) + trainer.save_checkpoint("tmp.ckpt") + to_lightning_cloud( + model_name, + checkpoint_path="tmp.ckpt", + api_key=os.getenv("API_KEY", ""), + project_id=os.getenv("PROJECT_ID", ""), + ) + + download_from_lightning_cloud(f"{username}/{model_name}") + assert os.path.isdir(os.path.join(LIGHTNING_STORAGE_DIR, username, model_name, version)) + + ckpt = load_from_lightning_cloud(f"{username}/{model_name}", load_checkpoint=True, model=model) + assert ckpt is not None diff --git a/tests/tests_app/model2cloud/test_requirements.py b/tests/tests_app/model2cloud/test_requirements.py new file mode 100644 index 0000000000000..4b91d563f5165 --- /dev/null +++ b/tests/tests_app/model2cloud/test_requirements.py @@ -0,0 +1,73 @@ +import os + +from tests_app.model2cloud.utils import cleanup + +from lightning_app.model2cloud import download_from_lightning_cloud, to_lightning_cloud +from pytorch_lightning.demos.boring_classes import BoringModel + +if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): + from tests_app.model2cloud.constants import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR +else: + from lightning_app.model2cloud.utils import LIGHTNING_STORAGE_DIR + + +def test_requirements_as_a_file(): + cleanup() + username = os.getenv("API_USERNAME", "") + if not username: + raise ValueError( + "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" + ) + + requirements_file_path = "tests/requirements.txt" + version = "latest" + model_name = "boring_model" + + to_lightning_cloud( + model_name, + version=version, + model=BoringModel(), + requirements_file_path=requirements_file_path, + api_key=os.getenv("API_KEY", ""), + project_id=os.getenv("PROJECT_ID", ""), + ) + + download_from_lightning_cloud(f"{username}/{model_name}") + + req_folder_path = os.path.join(LIGHTNING_STORAGE_DIR, username, model_name, version) + assert os.path.isdir(req_folder_path) + assert "requirements.txt" in os.listdir(req_folder_path) + + +def test_requirements_as_a_list(): + cleanup() + username = os.getenv("API_USERNAME", "") + if not username: + raise ValueError( + "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" + ) + + version = "1.0.0" + model_name = "boring_model" + requirements_list = ["pytorch_lightning==1.7.7", "lightning"] + + to_lightning_cloud( + model_name, + version=version, + model=BoringModel(), + requirements=requirements_list, + api_key=os.getenv("API_KEY", ""), + project_id=os.getenv("PROJECT_ID", ""), + ) + + download_from_lightning_cloud(f"{username}/{model_name}", version=version) + + req_folder_path = os.path.join(LIGHTNING_STORAGE_DIR, username, model_name, version) + assert os.path.isdir(req_folder_path) + assert "requirements.txt" in os.listdir(req_folder_path) + + with open(f"{req_folder_path}/requirements.txt") as req_file: + reqs = req_file.readlines() + reqs = [req.strip("\n") for req in reqs] + + assert requirements_list == reqs diff --git a/tests/tests_app/model2cloud/test_source_code.py b/tests/tests_app/model2cloud/test_source_code.py new file mode 100644 index 0000000000000..b502979aa5526 --- /dev/null +++ b/tests/tests_app/model2cloud/test_source_code.py @@ -0,0 +1,168 @@ +import inspect +import os +import tempfile + +from tests_app.model2cloud.utils import cleanup + +from lightning_app.model2cloud import download_from_lightning_cloud, to_lightning_cloud +from pytorch_lightning.demos.boring_classes import BoringModel + +if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): + from tests_app.model2cloud.constants import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR +else: + from lightning_app.model2cloud import LIGHTNING_STORAGE_DIR + + +def test_source_code_implicit(): + cleanup() + username = os.getenv("API_USERNAME", "") + if not username: + raise ValueError( + "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" + ) + + username = os.getenv("API_USERNAME") + model_name = "model_test_source_code_implicit" + to_lightning_cloud( + model_name, + model=BoringModel(), + api_key=os.getenv("API_KEY", ""), + project_id=os.getenv("PROJECT_ID", ""), + ) + + download_from_lightning_cloud(f"{username}/{model_name}") + assert os.path.isfile( + os.path.join( + LIGHTNING_STORAGE_DIR, + username, + model_name, + "latest", + str(os.path.basename(inspect.getsourcefile(BoringModel))), + ) + ) + + +def test_source_code_saving_disabled(): + cleanup() + username = os.getenv("API_USERNAME", "") + if not username: + raise ValueError( + "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" + ) + + model_name = "model_test_source_code_dont_save" + to_lightning_cloud( + model_name, + model=BoringModel(), + api_key=os.getenv("API_KEY", ""), + project_id=os.getenv("PROJECT_ID", ""), + save_code=False, + ) + + download_from_lightning_cloud(f"{username}/{model_name}") + assert not os.path.isfile( + os.path.join( + LIGHTNING_STORAGE_DIR, + username, + model_name, + "latest", + str(os.path.basename(inspect.getsourcefile(BoringModel))), + ) + ) + + +def test_source_code_explicit_relative_folder(): + cleanup() + username = os.getenv("API_USERNAME", "") + if not username: + raise ValueError( + "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" + ) + + username = os.getenv("API_USERNAME") + model_name = "model_test_source_code_explicit_relative" + dir_upload_path = f"../{os.path.basename(os.getcwd())}/tests/" + to_lightning_cloud( + model_name, + model=BoringModel(), + source_code_path=dir_upload_path, + api_key=os.getenv("API_KEY", ""), + project_id=os.getenv("PROJECT_ID", ""), + ) + + download_from_lightning_cloud(f"{username}/{model_name}") + + assert os.path.isdir( + os.path.join( + LIGHTNING_STORAGE_DIR, + username, + model_name, + "latest", + os.path.basename(os.path.abspath(dir_upload_path)), + ) + ) + + +def test_source_code_explicit_absolute_folder(): + cleanup() + username = os.getenv("API_USERNAME", "") + if not username: + raise ValueError( + "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" + ) + + username = os.getenv("API_USERNAME") + model_name = "model_test_source_code_explicit_absolute_path" + with tempfile.TemporaryDirectory() as tmpdir: + dir_upload_path = os.path.abspath(tmpdir) + to_lightning_cloud( + model_name, + model=BoringModel(), + source_code_path=dir_upload_path, + api_key=os.getenv("API_KEY", ""), + project_id=os.getenv("PROJECT_ID", ""), + ) + + download_from_lightning_cloud(f"{username}/{model_name}") + + assert os.path.isdir( + os.path.join( + LIGHTNING_STORAGE_DIR, + username, + model_name, + "latest", + os.path.basename(os.path.abspath(dir_upload_path)), + ) + ) + + +def test_source_code_explicit_file(): + cleanup() + username = os.getenv("API_USERNAME", "") + if not username: + raise ValueError( + "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" + ) + + username = os.getenv("API_USERNAME") + model_name = "model_test_source_code_explicit_file" + file_name = os.path.abspath("setup.py") + to_lightning_cloud( + model_name, + model=BoringModel(), + source_code_path=file_name, + api_key=os.getenv("API_KEY", ""), + project_id=os.getenv("PROJECT_ID", ""), + ) + + download_from_lightning_cloud(f"{username}/{model_name}") + + assert os.path.isfile( + os.path.join( + LIGHTNING_STORAGE_DIR, + username, + model_name, + "latest", + os.path.basename(file_name), + ) + ) diff --git a/tests/tests_app/model2cloud/test_versioning.py b/tests/tests_app/model2cloud/test_versioning.py new file mode 100644 index 0000000000000..0b43880e96dc6 --- /dev/null +++ b/tests/tests_app/model2cloud/test_versioning.py @@ -0,0 +1,100 @@ +import os +import platform + +import pytest +from tests_app.model2cloud.utils import cleanup + +from lightning_app.model2cloud.cloud_api import download_from_lightning_cloud, to_lightning_cloud +from pytorch_lightning.demos.boring_classes import BoringModel + +if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): + from tests_app.model2cloud.constants import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR +else: + from lightning_app.model2cloud.utils import LIGHTNING_STORAGE_DIR + + +def assert_download_successful(username, model_name, version): + folder_name = os.path.join( + LIGHTNING_STORAGE_DIR, + username, + model_name, + version, + ) + assert os.path.isdir(folder_name), f"Folder name: {folder_name} doesn't exist." + assert len(os.listdir(folder_name)) != 0 + + +@pytest.mark.parametrize( + ( + "case", + "expected_case", + ), + ( + [ + ("1.0.0", "version_1_0_0"), + ("0.0.1", "version_0_0_1"), + ("latest", "version_latest"), + ("1.0", "version_1_0"), + ("1", "version_1"), + ("0.1", "version_0_1"), + ("", "version_latest"), + ] + ), +) +def test_versioning_valid_case(case, expected_case): + cleanup() + username = os.getenv("API_USERNAME", "") + if not username: + raise ValueError( + "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" + ) + + model_name = "boring_model_versioning" + + api_key = os.getenv("API_KEY", "") + to_lightning_cloud( + model_name, + version=case, + model=BoringModel(), + api_key=api_key, + project_id=os.getenv("PROJECT_ID", ""), + ) + download_from_lightning_cloud(f"{username}/{model_name}", version=case) + assert_download_successful(username, model_name, expected_case) + + +@pytest.mark.parametrize( + "case", + ( + [ + " version with spaces ", + "*", + # "#", <-- TODO: Add it back later + "¡", + "©", + ] + ), +) +def test_versioning_invalid_case(case): + cleanup() + username = os.getenv("API_USERNAME", "") + if not username: + raise ValueError( + "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" + ) + + model_name = "boring_model_versioning" + + with pytest.raises(AssertionError): + api_key = os.getenv("API_KEY", "") + to_lightning_cloud( + model_name, + version=case, + model=BoringModel(), + api_key=api_key, + project_id=os.getenv("PROJECT_ID", ""), + ) + + error = OSError if case == "*" and platform.system() == "Windows" else AssertionError + with pytest.raises(error): + download_from_lightning_cloud(f"{username}/{model_name}", version=case) diff --git a/tests/tests_app/model2cloud/utils.py b/tests/tests_app/model2cloud/utils.py new file mode 100644 index 0000000000000..19faea756f322 --- /dev/null +++ b/tests/tests_app/model2cloud/utils.py @@ -0,0 +1,10 @@ +import os +import shutil + +from tests_app.model2cloud.constants import LIGHTNING_TEST_STORAGE_DIR + + +def cleanup(): + if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): + if os.path.isdir(LIGHTNING_TEST_STORAGE_DIR): + shutil.rmtree(LIGHTNING_TEST_STORAGE_DIR) From 3a820d72d7bfb5101a125ac3535c0b4167cabfac Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 30 Dec 2022 15:00:39 +0100 Subject: [PATCH 03/42] move > lai --- .gitignore | 1 - src/{lightning_app => lightning}/model2cloud/__init__.py | 0 .../model2cloud/authentication.py | 2 +- src/{lightning_app => lightning}/model2cloud/cloud_api.py | 8 ++++---- src/{lightning_app => lightning}/model2cloud/save.py | 2 +- src/{lightning_app => lightning}/model2cloud/utils.py | 0 tests/{tests_app => tests_lit}/model2cloud/__init__.py | 0 tests/{tests_app => tests_lit}/model2cloud/constants.py | 0 tests/{tests_app => tests_lit}/model2cloud/test_model.py | 2 +- .../model2cloud/test_requirements.py | 4 ++-- .../model2cloud/test_source_code.py | 4 ++-- .../model2cloud/test_versioning.py | 4 ++-- tests/{tests_app => tests_lit}/model2cloud/utils.py | 0 13 files changed, 13 insertions(+), 14 deletions(-) rename src/{lightning_app => lightning}/model2cloud/__init__.py (100%) rename src/{lightning_app => lightning}/model2cloud/authentication.py (96%) rename src/{lightning_app => lightning}/model2cloud/cloud_api.py (98%) rename src/{lightning_app => lightning}/model2cloud/save.py (99%) rename src/{lightning_app => lightning}/model2cloud/utils.py (100%) rename tests/{tests_app => tests_lit}/model2cloud/__init__.py (100%) rename tests/{tests_app => tests_lit}/model2cloud/constants.py (100%) rename tests/{tests_app => tests_lit}/model2cloud/test_model.py (98%) rename tests/{tests_app => tests_lit}/model2cloud/test_requirements.py (94%) rename tests/{tests_app => tests_lit}/model2cloud/test_source_code.py (97%) rename tests/{tests_app => tests_lit}/model2cloud/test_versioning.py (95%) rename tests/{tests_app => tests_lit}/model2cloud/utils.py (100%) diff --git a/.gitignore b/.gitignore index fbe0c29c29eca..a34923c07037c 100644 --- a/.gitignore +++ b/.gitignore @@ -48,7 +48,6 @@ wheels/ *.egg-info/ .installed.cfg *.egg -src/lightning/*/ src/*/version.info # PyInstaller diff --git a/src/lightning_app/model2cloud/__init__.py b/src/lightning/model2cloud/__init__.py similarity index 100% rename from src/lightning_app/model2cloud/__init__.py rename to src/lightning/model2cloud/__init__.py diff --git a/src/lightning_app/model2cloud/authentication.py b/src/lightning/model2cloud/authentication.py similarity index 96% rename from src/lightning_app/model2cloud/authentication.py rename to src/lightning/model2cloud/authentication.py index 5622283bc4050..7d5f93c082727 100644 --- a/src/lightning_app/model2cloud/authentication.py +++ b/src/lightning/model2cloud/authentication.py @@ -4,8 +4,8 @@ import requests from requests.models import HTTPBasicAuth +from lightning.app.model2cloud.utils import LIGHTNING_CLOUD_URL from lightning.app.utilities.network import LightningClient -from lightning_app.model2cloud.utils import LIGHTNING_CLOUD_URL def get_user_details(): diff --git a/src/lightning_app/model2cloud/cloud_api.py b/src/lightning/model2cloud/cloud_api.py similarity index 98% rename from src/lightning_app/model2cloud/cloud_api.py rename to src/lightning/model2cloud/cloud_api.py index 74a2cd1f61434..5c6f7908c3128 100644 --- a/src/lightning_app/model2cloud/cloud_api.py +++ b/src/lightning/model2cloud/cloud_api.py @@ -8,8 +8,8 @@ import requests import torch -from lightning_app.model2cloud.authentication import authenticate -from lightning_app.model2cloud.save import ( +from lightning.app.model2cloud.authentication import authenticate +from lightning.app.model2cloud.save import ( _download_and_extract_data_to, _save_checkpoint_from_path, _save_meta_data, @@ -21,7 +21,7 @@ _write_and_save_requirements, get_linked_output_dir, ) -from lightning_app.model2cloud.utils import ( +from lightning.app.model2cloud.utils import ( get_model_data, LIGHTNING_CLOUD_URL, LIGHTNING_STORAGE_FILE, @@ -32,7 +32,7 @@ if os.getenv("LIGHTNING_MODEL_STORE_TESTING", 0): from tests.constants import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR else: - from lightning_app.model2cloud.utils import LIGHTNING_STORAGE_DIR + from lightning.app.model2cloud.utils import LIGHTNING_STORAGE_DIR import lightning as L import pytorch_lightning as PL diff --git a/src/lightning_app/model2cloud/save.py b/src/lightning/model2cloud/save.py similarity index 99% rename from src/lightning_app/model2cloud/save.py rename to src/lightning/model2cloud/save.py index 9d4ffb28a08ff..de1b35d4dd46d 100644 --- a/src/lightning_app/model2cloud/save.py +++ b/src/lightning/model2cloud/save.py @@ -12,7 +12,7 @@ from tqdm import tqdm from tqdm.utils import CallbackIOWrapper -from lightning_app.model2cloud.utils import LIGHTNING_CLOUD_URL +from lightning.app.model2cloud.utils import LIGHTNING_CLOUD_URL logging.basicConfig(level=logging.INFO) diff --git a/src/lightning_app/model2cloud/utils.py b/src/lightning/model2cloud/utils.py similarity index 100% rename from src/lightning_app/model2cloud/utils.py rename to src/lightning/model2cloud/utils.py diff --git a/tests/tests_app/model2cloud/__init__.py b/tests/tests_lit/model2cloud/__init__.py similarity index 100% rename from tests/tests_app/model2cloud/__init__.py rename to tests/tests_lit/model2cloud/__init__.py diff --git a/tests/tests_app/model2cloud/constants.py b/tests/tests_lit/model2cloud/constants.py similarity index 100% rename from tests/tests_app/model2cloud/constants.py rename to tests/tests_lit/model2cloud/constants.py diff --git a/tests/tests_app/model2cloud/test_model.py b/tests/tests_lit/model2cloud/test_model.py similarity index 98% rename from tests/tests_app/model2cloud/test_model.py rename to tests/tests_lit/model2cloud/test_model.py index b1c78aeeb472d..5f0cf70d06558 100644 --- a/tests/tests_app/model2cloud/test_model.py +++ b/tests/tests_lit/model2cloud/test_model.py @@ -9,7 +9,7 @@ if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): from tests_app.model2cloud.constants import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR else: - from lightning_app.model2cloud.utils import LIGHTNING_STORAGE_DIR + from lightning.app.model2cloud.utils import LIGHTNING_STORAGE_DIR def test_model(): diff --git a/tests/tests_app/model2cloud/test_requirements.py b/tests/tests_lit/model2cloud/test_requirements.py similarity index 94% rename from tests/tests_app/model2cloud/test_requirements.py rename to tests/tests_lit/model2cloud/test_requirements.py index 4b91d563f5165..e857c01dba6fa 100644 --- a/tests/tests_app/model2cloud/test_requirements.py +++ b/tests/tests_lit/model2cloud/test_requirements.py @@ -2,13 +2,13 @@ from tests_app.model2cloud.utils import cleanup -from lightning_app.model2cloud import download_from_lightning_cloud, to_lightning_cloud +from lightning.app.model2cloud import download_from_lightning_cloud, to_lightning_cloud from pytorch_lightning.demos.boring_classes import BoringModel if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): from tests_app.model2cloud.constants import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR else: - from lightning_app.model2cloud.utils import LIGHTNING_STORAGE_DIR + from lightning.app.model2cloud.utils import LIGHTNING_STORAGE_DIR def test_requirements_as_a_file(): diff --git a/tests/tests_app/model2cloud/test_source_code.py b/tests/tests_lit/model2cloud/test_source_code.py similarity index 97% rename from tests/tests_app/model2cloud/test_source_code.py rename to tests/tests_lit/model2cloud/test_source_code.py index b502979aa5526..66cfbc48f128b 100644 --- a/tests/tests_app/model2cloud/test_source_code.py +++ b/tests/tests_lit/model2cloud/test_source_code.py @@ -4,13 +4,13 @@ from tests_app.model2cloud.utils import cleanup -from lightning_app.model2cloud import download_from_lightning_cloud, to_lightning_cloud +from lightning.app.model2cloud import download_from_lightning_cloud, to_lightning_cloud from pytorch_lightning.demos.boring_classes import BoringModel if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): from tests_app.model2cloud.constants import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR else: - from lightning_app.model2cloud import LIGHTNING_STORAGE_DIR + from lightning.app.model2cloud import LIGHTNING_STORAGE_DIR def test_source_code_implicit(): diff --git a/tests/tests_app/model2cloud/test_versioning.py b/tests/tests_lit/model2cloud/test_versioning.py similarity index 95% rename from tests/tests_app/model2cloud/test_versioning.py rename to tests/tests_lit/model2cloud/test_versioning.py index 0b43880e96dc6..5a5d2a2dc728f 100644 --- a/tests/tests_app/model2cloud/test_versioning.py +++ b/tests/tests_lit/model2cloud/test_versioning.py @@ -4,13 +4,13 @@ import pytest from tests_app.model2cloud.utils import cleanup -from lightning_app.model2cloud.cloud_api import download_from_lightning_cloud, to_lightning_cloud +from lightning.app.model2cloud.cloud_api import download_from_lightning_cloud, to_lightning_cloud from pytorch_lightning.demos.boring_classes import BoringModel if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): from tests_app.model2cloud.constants import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR else: - from lightning_app.model2cloud.utils import LIGHTNING_STORAGE_DIR + from lightning.app.model2cloud.utils import LIGHTNING_STORAGE_DIR def assert_download_successful(username, model_name, version): diff --git a/tests/tests_app/model2cloud/utils.py b/tests/tests_lit/model2cloud/utils.py similarity index 100% rename from tests/tests_app/model2cloud/utils.py rename to tests/tests_lit/model2cloud/utils.py From 984755758dc3c8a1dbae13fb8887446ce5dbec72 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 20 Jan 2023 09:49:05 +0100 Subject: [PATCH 04/42] cleaning --- src/lightning/model2cloud/__init__.py | 3 +++ src/lightning/model2cloud/authentication.py | 2 +- src/lightning/model2cloud/cloud_api.py | 16 +++++----------- src/lightning/model2cloud/save.py | 2 +- .../constants.py => tests_cloud/__init__.py} | 0 .../model2cloud => tests_cloud}/test_model.py | 8 ++++---- .../test_requirements.py | 8 ++++---- .../test_source_code.py | 8 ++++---- .../test_versioning.py | 8 ++++---- .../model2cloud => tests_cloud}/utils.py | 2 +- tests/tests_lit/model2cloud/__init__.py | 0 11 files changed, 27 insertions(+), 30 deletions(-) rename tests/{tests_lit/model2cloud/constants.py => tests_cloud/__init__.py} (100%) rename tests/{tests_lit/model2cloud => tests_cloud}/test_model.py (92%) rename tests/{tests_lit/model2cloud => tests_cloud}/test_requirements.py (87%) rename tests/{tests_lit/model2cloud => tests_cloud}/test_source_code.py (94%) rename tests/{tests_lit/model2cloud => tests_cloud}/test_versioning.py (89%) rename tests/{tests_lit/model2cloud => tests_cloud}/utils.py (73%) delete mode 100644 tests/tests_lit/model2cloud/__init__.py diff --git a/src/lightning/model2cloud/__init__.py b/src/lightning/model2cloud/__init__.py index e69de29bb2d1d..73157fec899e0 100644 --- a/src/lightning/model2cloud/__init__.py +++ b/src/lightning/model2cloud/__init__.py @@ -0,0 +1,3 @@ +from lightning.model2cloud.cloud_api import download_from_lightning_cloud, load_from_lightning_cloud, to_lightning_cloud + +__all__ = ["download_from_lightning_cloud", "load_from_lightning_cloud", "to_lightning_cloud"] diff --git a/src/lightning/model2cloud/authentication.py b/src/lightning/model2cloud/authentication.py index 7d5f93c082727..28b0ffffe236e 100644 --- a/src/lightning/model2cloud/authentication.py +++ b/src/lightning/model2cloud/authentication.py @@ -4,8 +4,8 @@ import requests from requests.models import HTTPBasicAuth -from lightning.app.model2cloud.utils import LIGHTNING_CLOUD_URL from lightning.app.utilities.network import LightningClient +from lightning.model2cloud.utils import LIGHTNING_CLOUD_URL def get_user_details(): diff --git a/src/lightning/model2cloud/cloud_api.py b/src/lightning/model2cloud/cloud_api.py index 5c6f7908c3128..5951936abd812 100644 --- a/src/lightning/model2cloud/cloud_api.py +++ b/src/lightning/model2cloud/cloud_api.py @@ -8,8 +8,8 @@ import requests import torch -from lightning.app.model2cloud.authentication import authenticate -from lightning.app.model2cloud.save import ( +from lightning.model2cloud.authentication import authenticate +from lightning.model2cloud.save import ( _download_and_extract_data_to, _save_checkpoint_from_path, _save_meta_data, @@ -21,18 +21,12 @@ _write_and_save_requirements, get_linked_output_dir, ) -from lightning.app.model2cloud.utils import ( - get_model_data, - LIGHTNING_CLOUD_URL, - LIGHTNING_STORAGE_FILE, - split_name, - stage, -) +from lightning.model2cloud.utils import get_model_data, LIGHTNING_CLOUD_URL, LIGHTNING_STORAGE_FILE, split_name, stage if os.getenv("LIGHTNING_MODEL_STORE_TESTING", 0): - from tests.constants import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR + from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR else: - from lightning.app.model2cloud.utils import LIGHTNING_STORAGE_DIR + from lightning.model2cloud.utils import LIGHTNING_STORAGE_DIR import lightning as L import pytorch_lightning as PL diff --git a/src/lightning/model2cloud/save.py b/src/lightning/model2cloud/save.py index de1b35d4dd46d..13747bf13d99a 100644 --- a/src/lightning/model2cloud/save.py +++ b/src/lightning/model2cloud/save.py @@ -12,7 +12,7 @@ from tqdm import tqdm from tqdm.utils import CallbackIOWrapper -from lightning.app.model2cloud.utils import LIGHTNING_CLOUD_URL +from lightning.model2cloud.utils import LIGHTNING_CLOUD_URL logging.basicConfig(level=logging.INFO) diff --git a/tests/tests_lit/model2cloud/constants.py b/tests/tests_cloud/__init__.py similarity index 100% rename from tests/tests_lit/model2cloud/constants.py rename to tests/tests_cloud/__init__.py diff --git a/tests/tests_lit/model2cloud/test_model.py b/tests/tests_cloud/test_model.py similarity index 92% rename from tests/tests_lit/model2cloud/test_model.py rename to tests/tests_cloud/test_model.py index 5f0cf70d06558..113554f79fade 100644 --- a/tests/tests_lit/model2cloud/test_model.py +++ b/tests/tests_cloud/test_model.py @@ -1,15 +1,15 @@ import os -from models_cloud import download_from_lightning_cloud, load_from_lightning_cloud, to_lightning_cloud -from tests.utils import cleanup +from tests_cloud.utils import cleanup import pytorch_lightning as pl +from lightning.model2cloud import download_from_lightning_cloud, load_from_lightning_cloud, to_lightning_cloud from pytorch_lightning.demos.boring_classes import BoringModel if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): - from tests_app.model2cloud.constants import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR + from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR else: - from lightning.app.model2cloud.utils import LIGHTNING_STORAGE_DIR + from lightning.model2cloud.utils import LIGHTNING_STORAGE_DIR def test_model(): diff --git a/tests/tests_lit/model2cloud/test_requirements.py b/tests/tests_cloud/test_requirements.py similarity index 87% rename from tests/tests_lit/model2cloud/test_requirements.py rename to tests/tests_cloud/test_requirements.py index e857c01dba6fa..7ea2fef9df9e2 100644 --- a/tests/tests_lit/model2cloud/test_requirements.py +++ b/tests/tests_cloud/test_requirements.py @@ -1,14 +1,14 @@ import os -from tests_app.model2cloud.utils import cleanup +from tests_cloud.utils import cleanup -from lightning.app.model2cloud import download_from_lightning_cloud, to_lightning_cloud +from lightning.model2cloud import download_from_lightning_cloud, to_lightning_cloud from pytorch_lightning.demos.boring_classes import BoringModel if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): - from tests_app.model2cloud.constants import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR + from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR else: - from lightning.app.model2cloud.utils import LIGHTNING_STORAGE_DIR + from lightning.model2cloud.utils import LIGHTNING_STORAGE_DIR def test_requirements_as_a_file(): diff --git a/tests/tests_lit/model2cloud/test_source_code.py b/tests/tests_cloud/test_source_code.py similarity index 94% rename from tests/tests_lit/model2cloud/test_source_code.py rename to tests/tests_cloud/test_source_code.py index 66cfbc48f128b..ad889e45b563b 100644 --- a/tests/tests_lit/model2cloud/test_source_code.py +++ b/tests/tests_cloud/test_source_code.py @@ -2,15 +2,15 @@ import os import tempfile -from tests_app.model2cloud.utils import cleanup +from tests_cloud.utils import cleanup -from lightning.app.model2cloud import download_from_lightning_cloud, to_lightning_cloud +from lightning.model2cloud import download_from_lightning_cloud, to_lightning_cloud from pytorch_lightning.demos.boring_classes import BoringModel if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): - from tests_app.model2cloud.constants import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR + from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR else: - from lightning.app.model2cloud import LIGHTNING_STORAGE_DIR + from lightning.model2cloud import LIGHTNING_STORAGE_DIR def test_source_code_implicit(): diff --git a/tests/tests_lit/model2cloud/test_versioning.py b/tests/tests_cloud/test_versioning.py similarity index 89% rename from tests/tests_lit/model2cloud/test_versioning.py rename to tests/tests_cloud/test_versioning.py index 5a5d2a2dc728f..76660ea7295a8 100644 --- a/tests/tests_lit/model2cloud/test_versioning.py +++ b/tests/tests_cloud/test_versioning.py @@ -2,15 +2,15 @@ import platform import pytest -from tests_app.model2cloud.utils import cleanup +from tests_cloud.utils import cleanup -from lightning.app.model2cloud.cloud_api import download_from_lightning_cloud, to_lightning_cloud +from lightning.model2cloud.cloud_api import download_from_lightning_cloud, to_lightning_cloud from pytorch_lightning.demos.boring_classes import BoringModel if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): - from tests_app.model2cloud.constants import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR + from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR else: - from lightning.app.model2cloud.utils import LIGHTNING_STORAGE_DIR + from lightning.model2cloud.utils import LIGHTNING_STORAGE_DIR def assert_download_successful(username, model_name, version): diff --git a/tests/tests_lit/model2cloud/utils.py b/tests/tests_cloud/utils.py similarity index 73% rename from tests/tests_lit/model2cloud/utils.py rename to tests/tests_cloud/utils.py index 19faea756f322..8fd46e1b23740 100644 --- a/tests/tests_lit/model2cloud/utils.py +++ b/tests/tests_cloud/utils.py @@ -1,7 +1,7 @@ import os import shutil -from tests_app.model2cloud.constants import LIGHTNING_TEST_STORAGE_DIR +from tests_cloud import LIGHTNING_TEST_STORAGE_DIR def cleanup(): diff --git a/tests/tests_lit/model2cloud/__init__.py b/tests/tests_lit/model2cloud/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 From 701e2402a124a328c160cea92fc2d99124fd204c Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 20 Jan 2023 10:17:44 +0100 Subject: [PATCH 05/42] readme --- src/lightning/model2cloud/README.md | 79 +++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 src/lightning/model2cloud/README.md diff --git a/src/lightning/model2cloud/README.md b/src/lightning/model2cloud/README.md new file mode 100644 index 0000000000000..8026831283a7b --- /dev/null +++ b/src/lightning/model2cloud/README.md @@ -0,0 +1,79 @@ +## Getting Started + +- Login to lightning.ai (_optional_) \<-- takes less than a minute. ⏩ +- Store your models on the cloud \<-- simple call: `to_lightning_cloud(...)`. 🗳️ +- Share it with your friends \<-- just share the "username/model_name" (and version if required) format. :handshake: +- They download using a simple call: `download_from_lightning_cloud("username/model_name", version="your_version")`. :wink: +- They load your cool model. `load_from_lightning_cloud("username/model_name", version="your_version")`. :tada: +- Lightning :zap: fast, isn't it?. :heart: + +## Usage + +**Storing to the cloud** + +```python +from lightning.model2cloud import to_lightning_cloud +from sample.model import LitAutoEncoder, Encoder, Decoder + +# Initialize your model here +autoencoder = LitAutoEncoder(Encoder(), Decoder()) + +# Pass the model object: +# No need to pass the username (we'll deduce ourselves), just pass the model name you want as the first argument (with an optional version): +# format: `model_name:version` (version can either be latest or combination of digits and full-stops: 1.0.0 for example) +to_lightning_cloud("unique_model_mnist", model=autoencoder, source_code_path="sample") + +# version: +to_lightning_cloud( + "unique_model_mnist", + version="1.0.0", + model=autoencoder, + source_code_path="sample/model.py", +) + +# OR: (this will save the file which has the model defined) +to_lightning_cloud("krshrimali/unique_model_mnist", model=autoencoder) +``` + +You can also pass the checkpoint path: `to_lightning_cloud("model_name", version="latest", checkpoint_path=...)`. + +**Downloading from the cloud** + +```python +from lightning.model2cloud import download_from_lightning_cloud + +download_from_lightning_cloud( + "krshrimali/unique_model_mnist", output_dir="your_output_dir" +) +# OR: (default to lightning_model_storage $HOME/.lightning/lightning_model_store/username//version_/ folder) +download_from_lightning_cloud("krshrimali/unique_model_mnist") +``` + +**Loading model** + +```python +from lightning.model2cloud import load_from_lightning_cloud + +from ..version_. import LitAutoEncoder, Encoder, Decoder +model = load_from_lightning_cloud("/>", version="version") # version is optional (defaults to latest) + +# OR: load weights or checkpoint (if they were uploaded) +load_from_lightning_cloud("/", version="version", load_weights=True/False, load_checkpoint=True/False) +print(model) +``` + +**Loading model weights** + +```python +from lightning.model2cloud import load_from_lightning_cloud + +# If you had passed an `output_dir=...` to download_from_lightning_cloud(...), then you can just do: +from output_dir. import LitAutoEncoder, Encoder, Decoder + +model = LitAutoEncoder(Encoder(), Decoder()) + +model = load_from_lightning_cloud(load_weights=True, model=model) +print("State dict: ", model.state_dict()) +``` + +Loading checkpoint is similar, just do: `load_checkpoint=True`. From a2599667cf5e8993ea8b6b409288f8491a5ba3b1 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 20 Jan 2023 10:19:51 +0100 Subject: [PATCH 06/42] precommit --- src/lightning/model2cloud/README.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/lightning/model2cloud/README.md b/src/lightning/model2cloud/README.md index 8026831283a7b..71752c1c2d109 100644 --- a/src/lightning/model2cloud/README.md +++ b/src/lightning/model2cloud/README.md @@ -42,9 +42,7 @@ You can also pass the checkpoint path: `to_lightning_cloud("model_name", version ```python from lightning.model2cloud import download_from_lightning_cloud -download_from_lightning_cloud( - "krshrimali/unique_model_mnist", output_dir="your_output_dir" -) +download_from_lightning_cloud("krshrimali/unique_model_mnist", output_dir="your_output_dir") # OR: (default to lightning_model_storage $HOME/.lightning/lightning_model_store/username//version_/ folder) download_from_lightning_cloud("krshrimali/unique_model_mnist") ``` @@ -54,11 +52,15 @@ download_from_lightning_cloud("krshrimali/unique_model_mnist") ```python from lightning.model2cloud import load_from_lightning_cloud -from ..version_. import LitAutoEncoder, Encoder, Decoder -model = load_from_lightning_cloud("/>", version="version") # version is optional (defaults to latest) +# from ..version_. import LitAutoEncoder, Encoder, Decoder +model = load_from_lightning_cloud( + "/>", version="version" +) # version is optional (defaults to latest) # OR: load weights or checkpoint (if they were uploaded) -load_from_lightning_cloud("/", version="version", load_weights=True/False, load_checkpoint=True/False) +load_from_lightning_cloud( + "/", version="version", load_weights=True / False, load_checkpoint=True / False +) print(model) ``` @@ -68,7 +70,7 @@ print(model) from lightning.model2cloud import load_from_lightning_cloud # If you had passed an `output_dir=...` to download_from_lightning_cloud(...), then you can just do: -from output_dir. import LitAutoEncoder, Encoder, Decoder +# from output_dir. import LitAutoEncoder, Encoder, Decoder model = LitAutoEncoder(Encoder(), Decoder()) From 074c337693745027e734199eafe01d97ce07c165 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 20 Jan 2023 10:31:42 +0100 Subject: [PATCH 07/42] store --- src/lightning/model2cloud/__init__.py | 3 --- src/lightning/{model2cloud => store}/README.md | 8 ++++---- src/lightning/store/__init__.py | 3 +++ src/lightning/{model2cloud => store}/authentication.py | 2 +- src/lightning/{model2cloud => store}/cloud_api.py | 8 ++++---- src/lightning/{model2cloud => store}/save.py | 2 +- src/lightning/{model2cloud => store}/utils.py | 0 tests/tests_cloud/test_model.py | 4 ++-- tests/tests_cloud/test_requirements.py | 4 ++-- tests/tests_cloud/test_source_code.py | 4 ++-- tests/tests_cloud/test_versioning.py | 4 ++-- 11 files changed, 21 insertions(+), 21 deletions(-) delete mode 100644 src/lightning/model2cloud/__init__.py rename src/lightning/{model2cloud => store}/README.md (92%) create mode 100644 src/lightning/store/__init__.py rename src/lightning/{model2cloud => store}/authentication.py (96%) rename src/lightning/{model2cloud => store}/cloud_api.py (98%) rename src/lightning/{model2cloud => store}/save.py (99%) rename src/lightning/{model2cloud => store}/utils.py (100%) diff --git a/src/lightning/model2cloud/__init__.py b/src/lightning/model2cloud/__init__.py deleted file mode 100644 index 73157fec899e0..0000000000000 --- a/src/lightning/model2cloud/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from lightning.model2cloud.cloud_api import download_from_lightning_cloud, load_from_lightning_cloud, to_lightning_cloud - -__all__ = ["download_from_lightning_cloud", "load_from_lightning_cloud", "to_lightning_cloud"] diff --git a/src/lightning/model2cloud/README.md b/src/lightning/store/README.md similarity index 92% rename from src/lightning/model2cloud/README.md rename to src/lightning/store/README.md index 71752c1c2d109..522f4aef809a0 100644 --- a/src/lightning/model2cloud/README.md +++ b/src/lightning/store/README.md @@ -12,7 +12,7 @@ **Storing to the cloud** ```python -from lightning.model2cloud import to_lightning_cloud +from lightning.store import to_lightning_cloud from sample.model import LitAutoEncoder, Encoder, Decoder # Initialize your model here @@ -40,7 +40,7 @@ You can also pass the checkpoint path: `to_lightning_cloud("model_name", version **Downloading from the cloud** ```python -from lightning.model2cloud import download_from_lightning_cloud +from lightning.store import download_from_lightning_cloud download_from_lightning_cloud("krshrimali/unique_model_mnist", output_dir="your_output_dir") # OR: (default to lightning_model_storage $HOME/.lightning/lightning_model_store/username//version_/ folder) @@ -50,7 +50,7 @@ download_from_lightning_cloud("krshrimali/unique_model_mnist") **Loading model** ```python -from lightning.model2cloud import load_from_lightning_cloud +from lightning.store import load_from_lightning_cloud # from ..version_. import LitAutoEncoder, Encoder, Decoder model = load_from_lightning_cloud( @@ -67,7 +67,7 @@ print(model) **Loading model weights** ```python -from lightning.model2cloud import load_from_lightning_cloud +from lightning.store import load_from_lightning_cloud # If you had passed an `output_dir=...` to download_from_lightning_cloud(...), then you can just do: # from output_dir. import LitAutoEncoder, Encoder, Decoder diff --git a/src/lightning/store/__init__.py b/src/lightning/store/__init__.py new file mode 100644 index 0000000000000..31ce7ade8ef96 --- /dev/null +++ b/src/lightning/store/__init__.py @@ -0,0 +1,3 @@ +from lightning.store.cloud_api import download_from_lightning_cloud, load_from_lightning_cloud, to_lightning_cloud + +__all__ = ["download_from_lightning_cloud", "load_from_lightning_cloud", "to_lightning_cloud"] diff --git a/src/lightning/model2cloud/authentication.py b/src/lightning/store/authentication.py similarity index 96% rename from src/lightning/model2cloud/authentication.py rename to src/lightning/store/authentication.py index 28b0ffffe236e..b006f55aa668c 100644 --- a/src/lightning/model2cloud/authentication.py +++ b/src/lightning/store/authentication.py @@ -5,7 +5,7 @@ from requests.models import HTTPBasicAuth from lightning.app.utilities.network import LightningClient -from lightning.model2cloud.utils import LIGHTNING_CLOUD_URL +from lightning.store.utils import LIGHTNING_CLOUD_URL def get_user_details(): diff --git a/src/lightning/model2cloud/cloud_api.py b/src/lightning/store/cloud_api.py similarity index 98% rename from src/lightning/model2cloud/cloud_api.py rename to src/lightning/store/cloud_api.py index 5951936abd812..5e9c95870fd16 100644 --- a/src/lightning/model2cloud/cloud_api.py +++ b/src/lightning/store/cloud_api.py @@ -8,8 +8,8 @@ import requests import torch -from lightning.model2cloud.authentication import authenticate -from lightning.model2cloud.save import ( +from lightning.store.authentication import authenticate +from lightning.store.save import ( _download_and_extract_data_to, _save_checkpoint_from_path, _save_meta_data, @@ -21,12 +21,12 @@ _write_and_save_requirements, get_linked_output_dir, ) -from lightning.model2cloud.utils import get_model_data, LIGHTNING_CLOUD_URL, LIGHTNING_STORAGE_FILE, split_name, stage +from lightning.store.utils import get_model_data, LIGHTNING_CLOUD_URL, LIGHTNING_STORAGE_FILE, split_name, stage if os.getenv("LIGHTNING_MODEL_STORE_TESTING", 0): from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR else: - from lightning.model2cloud.utils import LIGHTNING_STORAGE_DIR + from lightning.store.utils import LIGHTNING_STORAGE_DIR import lightning as L import pytorch_lightning as PL diff --git a/src/lightning/model2cloud/save.py b/src/lightning/store/save.py similarity index 99% rename from src/lightning/model2cloud/save.py rename to src/lightning/store/save.py index 13747bf13d99a..df8bc2943f4b3 100644 --- a/src/lightning/model2cloud/save.py +++ b/src/lightning/store/save.py @@ -12,7 +12,7 @@ from tqdm import tqdm from tqdm.utils import CallbackIOWrapper -from lightning.model2cloud.utils import LIGHTNING_CLOUD_URL +from lightning.store.utils import LIGHTNING_CLOUD_URL logging.basicConfig(level=logging.INFO) diff --git a/src/lightning/model2cloud/utils.py b/src/lightning/store/utils.py similarity index 100% rename from src/lightning/model2cloud/utils.py rename to src/lightning/store/utils.py diff --git a/tests/tests_cloud/test_model.py b/tests/tests_cloud/test_model.py index 113554f79fade..a3a8fcbc2117d 100644 --- a/tests/tests_cloud/test_model.py +++ b/tests/tests_cloud/test_model.py @@ -3,13 +3,13 @@ from tests_cloud.utils import cleanup import pytorch_lightning as pl -from lightning.model2cloud import download_from_lightning_cloud, load_from_lightning_cloud, to_lightning_cloud +from lightning.store import download_from_lightning_cloud, load_from_lightning_cloud, to_lightning_cloud from pytorch_lightning.demos.boring_classes import BoringModel if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR else: - from lightning.model2cloud.utils import LIGHTNING_STORAGE_DIR + from lightning.store.utils import LIGHTNING_STORAGE_DIR def test_model(): diff --git a/tests/tests_cloud/test_requirements.py b/tests/tests_cloud/test_requirements.py index 7ea2fef9df9e2..571e4f5037705 100644 --- a/tests/tests_cloud/test_requirements.py +++ b/tests/tests_cloud/test_requirements.py @@ -2,13 +2,13 @@ from tests_cloud.utils import cleanup -from lightning.model2cloud import download_from_lightning_cloud, to_lightning_cloud +from lightning.store import download_from_lightning_cloud, to_lightning_cloud from pytorch_lightning.demos.boring_classes import BoringModel if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR else: - from lightning.model2cloud.utils import LIGHTNING_STORAGE_DIR + from lightning.store.utils import LIGHTNING_STORAGE_DIR def test_requirements_as_a_file(): diff --git a/tests/tests_cloud/test_source_code.py b/tests/tests_cloud/test_source_code.py index ad889e45b563b..68da410a5ddb9 100644 --- a/tests/tests_cloud/test_source_code.py +++ b/tests/tests_cloud/test_source_code.py @@ -4,13 +4,13 @@ from tests_cloud.utils import cleanup -from lightning.model2cloud import download_from_lightning_cloud, to_lightning_cloud +from lightning.store import download_from_lightning_cloud, to_lightning_cloud from pytorch_lightning.demos.boring_classes import BoringModel if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR else: - from lightning.model2cloud import LIGHTNING_STORAGE_DIR + from lightning.store import LIGHTNING_STORAGE_DIR def test_source_code_implicit(): diff --git a/tests/tests_cloud/test_versioning.py b/tests/tests_cloud/test_versioning.py index 76660ea7295a8..132f913068ad6 100644 --- a/tests/tests_cloud/test_versioning.py +++ b/tests/tests_cloud/test_versioning.py @@ -4,13 +4,13 @@ import pytest from tests_cloud.utils import cleanup -from lightning.model2cloud.cloud_api import download_from_lightning_cloud, to_lightning_cloud +from lightning.store.cloud_api import download_from_lightning_cloud, to_lightning_cloud from pytorch_lightning.demos.boring_classes import BoringModel if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR else: - from lightning.model2cloud.utils import LIGHTNING_STORAGE_DIR + from lightning.store.utils import LIGHTNING_STORAGE_DIR def assert_download_successful(username, model_name, version): From a086d8518a3b71ed608d0da0d2600a4825055814 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 20 Jan 2023 10:43:05 +0100 Subject: [PATCH 08/42] docs --- src/lightning/store/authentication.py | 14 +++ src/lightning/store/cloud_api.py | 171 ++++++++++++-------------- src/lightning/store/save.py | 14 +++ src/lightning/store/utils.py | 26 ++-- 4 files changed, 124 insertions(+), 101 deletions(-) diff --git a/src/lightning/store/authentication.py b/src/lightning/store/authentication.py index b006f55aa668c..aa2a78ba460b8 100644 --- a/src/lightning/store/authentication.py +++ b/src/lightning/store/authentication.py @@ -1,3 +1,17 @@ +# Copyright The Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import json import webbrowser diff --git a/src/lightning/store/cloud_api.py b/src/lightning/store/cloud_api.py index 5e9c95870fd16..fa2b72b192bd5 100644 --- a/src/lightning/store/cloud_api.py +++ b/src/lightning/store/cloud_api.py @@ -1,9 +1,23 @@ +# Copyright The Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import json import logging import os import sys import tempfile -from typing import List +from typing import List, Optional import requests import torch @@ -41,7 +55,7 @@ def to_lightning_cloud( source_code_path: str = "", checkpoint_path: str = "", requirements_file_path: str = "", - requirements: List[str] = [], + requirements: Optional[List[str]] = None, weights_only: bool = False, api_key: str = "", project_id: str = "", @@ -50,48 +64,44 @@ def to_lightning_cloud( *args, **kwargs, ): - """ - Parameters - ========== - - :param: name: (str) - The model name. Model/Checkpoint will be uploaded with this unique name. Format: "model_name" - :param: version: (str, default="latest") - The version of the model to be uploaded. If not provided, default will be latest (not overridden). - :param: model: (default=None) - The model object (initialized). This is optional, but if `checkpoint_path` is not passed, - it will raise an error. (Optional) - :param: source_code_path: (str, default="") - The path to the source code that needs to be uploaded along with the model. - The path can point to a python file or a directory. Path pointing to a non-python file - will raise an error. (Optional) - :param: checkpoint_path (str, default="") - The path to the checkpoint that needs to be uploaded. (Optional) - :param: requirements_file_path (str, default="") - The path to the requirements file, will always be uploaded with the name of `requirements.txt`. - :param: requirements (List[str], default=[]) - List of requirements as strings, that will be written as `requirements.txt` and - then uploaded. If both `requirements_file_path` and `requirements` are passed, - a warning is raised as `requirements` is given the priority over `requirements_file_path`. - :param: weights_only (bool, default=False) - If set to `True`, it will only save model weights and nothing else. This raises - an error if `weights_only` is `True` but no `model` is passed. - :param: api_key (str, default="") - API_KEY used for authentication. Fetch it after logging to https://lightning.ai - (in the keys tab in the settings). If not passed, the API will attempt to - either find the credentials in your system or opening the login prompt. - :param: project_id (str, default="") - Some users have multiple projects with unique `project_id`. They need to pass - this in order to upload models to the cloud. - :param: progress_bar (bool, default=True) - A progress bar to show the uploading status. Disable this if not needed, by setting to `False`. - :param: save_code (bool, default=True) - By default, the API saves the code where the model is defined. - Set it to `False` if saving code is not desired. - - Returns - ======== - None + """Store model to lightning cloud. + + Args: + + name: + The model name. Model/Checkpoint will be uploaded with this unique name. Format: "model_name" + version: + The version of the model to be uploaded. If not provided, default will be latest (not overridden). + model: + The model object (initialized). This is optional, but if `checkpoint_path` is not passed, + it will raise an error. (Optional) + source_code_path: + The path to the source code that needs to be uploaded along with the model. + The path can point to a python file or a directory. Path pointing to a non-python file + will raise an error. (Optional) + checkpoint_path: + The path to the checkpoint that needs to be uploaded. (Optional) + requirements_file_path: + The path to the requirements file, will always be uploaded with the name of `requirements.txt`. + requirements: + List of requirements as strings, that will be written as `requirements.txt` and + then uploaded. If both `requirements_file_path` and `requirements` are passed, + a warning is raised as `requirements` is given the priority over `requirements_file_path`. + weights_only: + If set to `True`, it will only save model weights and nothing else. This raises + an error if `weights_only` is `True` but no `model` is passed. + api_key: + API_KEY used for authentication. Fetch it after logging to https://lightning.ai + (in the keys tab in the settings). If not passed, the API will attempt to + either find the credentials in your system or opening the login prompt. + project_id: + Some users have multiple projects with unique `project_id`. They need to pass + this in order to upload models to the cloud. + progress_bar: + A progress bar to show the uploading status. Disable this if not needed, by setting to `False`. + save_code: + By default, the API saves the code where the model is defined. + Set it to `False` if saving code is not desired. """ if model is None and checkpoint_path is None: raise ValueError( @@ -113,38 +123,21 @@ def to_lightning_cloud( _, model_name, _ = split_name(name, version=version, l_stage=stage.UPLOAD) username_from_api_key, api_key = authenticate(api_key) - name = f"{username_from_api_key}/{model_name}:{version}" + # name = f"{username_from_api_key}/{model_name}:{version}" stored = {} with tempfile.TemporaryDirectory() as tmpdir: if checkpoint_path: - stored = _save_checkpoint_from_path( - model_name, - path=checkpoint_path, - tmpdir=tmpdir, - stored=stored, - ) + stored = _save_checkpoint_from_path(model_name, path=checkpoint_path, tmpdir=tmpdir, stored=stored) if model: stored = _save_model_weights( - model_name, - model_state_dict=model.state_dict(), - tmpdir=tmpdir, - stored=stored, - *args, - **kwargs, + model_name, model_state_dict=model.state_dict(), tmpdir=tmpdir, stored=stored, *args, **kwargs ) if not weights_only: - stored = _save_model( - model_name, - model=model, - tmpdir=tmpdir, - stored=stored, - *args, - **kwargs, - ) + stored = _save_model(model_name, model=model, tmpdir=tmpdir, stored=stored, *args, **kwargs) if save_code: stored = _save_model_code( @@ -252,21 +245,17 @@ def download_from_lightning_cloud( output_dir: str = "", progress_bar: bool = True, ): - """ - Parameters - ========== - :param: name (str): - The unique name of the model to be downloaded. Format: `/`. - :param: version: (str, default="latest") - The version of the model to be uploaded. If not provided, default will be latest (not overridden). - :param: output_dir (str, default=""): - The target directory, where the model and other data will be stored. If not passed, + """Download model from lightning cloud. + + Args: + name: + The unique name of the model to be downloaded. Format: `/`. + version: + The version of the model to be uploaded. If not provided, default will be latest (not overridden). + output_dir: + The target directory, where the model and other data will be stored. If not passed, the data will be stored in `$HOME/.lightning/lightning_model_store///`. (`version` defaults to `latest`) - - Returns - ======= - None """ version = version or "latest" username, model_name, version = split_name(name, version=version, l_stage=stage.DOWNLOAD) @@ -347,21 +336,17 @@ def load_from_lightning_cloud( *args, **kwargs, ): - """ - Parameters - ========== - :param: name (str) - Name of the model to load. Format: `/` - :param: version: (str, default="latest") - The version of the model to be uploaded. If not provided, default will be latest (not overridden). - :param: load_weights (bool, default=False) - Loads only weights if this is set to `True`. Needs `model` to be passed in order to load the weights. - :param: load_checkpoint (bool, default=False) - Loads checkpoint if this is set to `True`. Only a `LightningModule` model is supported for this feature. - - Returns - ======= - None + """Load model from lightning cloud. + + Args: + name: + Name of the model to load. Format: `/` + version: + The version of the model to be uploaded. If not provided, default will be latest (not overridden). + load_weights: + Loads only weights if this is set to `True`. Needs `model` to be passed in order to load the weights. + load_checkpoint: + Loads checkpoint if this is set to `True`. Only a `LightningModule` model is supported for this feature. """ if load_weights and load_checkpoint: raise ValueError( diff --git a/src/lightning/store/save.py b/src/lightning/store/save.py index df8bc2943f4b3..a50ddaa1e4de1 100644 --- a/src/lightning/store/save.py +++ b/src/lightning/store/save.py @@ -1,3 +1,17 @@ +# Copyright The Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import inspect import json import logging diff --git a/src/lightning/store/utils.py b/src/lightning/store/utils.py index 03bef838582b0..99395323e5806 100644 --- a/src/lightning/store/utils.py +++ b/src/lightning/store/utils.py @@ -1,3 +1,17 @@ +# Copyright The Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import json import os from enum import Enum @@ -9,9 +23,9 @@ class stage(Enum): DOWNLOAD = 2 -LIGHTNING_DIR = f"{os.path.expanduser('~')}/.lightning" -LIGHTNING_STORAGE_FILE = f"{LIGHTNING_DIR}/.lightning_model_storage" -LIGHTNING_STORAGE_DIR = f"{LIGHTNING_DIR}/lightning_model_store" +LIGHTNING_DIR = os.path.join(os.path.expanduser("~"), ".lightning") +LIGHTNING_STORAGE_FILE = os.path.join(LIGHTNING_DIR, ".lightning_model_storage") +LIGHTNING_STORAGE_DIR = os.path.join(LIGHTNING_DIR, "lightning_model_store") LIGHTNING_CLOUD_URL = os.getenv("LIGHTNING_CLOUD_URL", default="https://lightning.ai") @@ -38,11 +52,7 @@ def _split_name(name: str, version: str, l_stage: stage): def split_name(name: str, version: str, l_stage: stage): username, model_name, version = _split_name(name, version, l_stage) - return ( - username, - model_name, - version, - ) + return (username, model_name, version) def get_model_data(name: str, version: str): From a411488ef7ee4976820bac9e9a1edc6092a464d2 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 20 Jan 2023 13:20:15 +0100 Subject: [PATCH 09/42] ci --- .github/workflows/ci-tests-cloud.yml | 101 +++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 .github/workflows/ci-tests-cloud.yml diff --git a/.github/workflows/ci-tests-cloud.yml b/.github/workflows/ci-tests-cloud.yml new file mode 100644 index 0000000000000..3bfb26c1d07a2 --- /dev/null +++ b/.github/workflows/ci-tests-cloud.yml @@ -0,0 +1,101 @@ +name: Test Cloud + +# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows +on: + push: + branches: [master, "release/*"] + pull_request: + branches: [master, "release/*"] + types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped + paths: + - ".actions/**" + - ".github/workflows/ci-tests-cloud.yml" + - "src/lightning/store/**" + - "tests/tests_cloud/**" + - "setup.py" + - "!*.md" + - "!**/*.md" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} + +defaults: + run: + shell: bash + +jobs: + app-pytest: + if: github.event.pull_request.draft == false + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04, macOS-11, windows-2022] + python-version: ["3.8"] + requires: ["oldest", "latest"] + + # Timeout: https://stackoverflow.com/a/59076067/4521646 + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: basic setup + run: pip install -q -r .actions/requirements.txt + + - name: Set min. dependencies + if: ${{ matrix.requires == 'oldest' }} + run: python .actions/assistant.py replace_oldest_ver + + - name: Get pip cache dir + id: pip-cache + run: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT + + - name: Cache pip + uses: actions/cache@v3 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ hashFiles('requirements/app/base.txt') }} + restore-keys: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.requires }}- + + - name: Install package & dependencies + env: + FREEZE_REQUIREMENTS: 1 + TORCH_URL: https://download.pytorch.org/whl/cpu/torch_stable.html + run: | + pip install -e .[dev] --upgrade --find-links ${TORCH_URL} + pip list + + - name: Tests + working-directory: ./tests + env: + API_KEY: ${{ secrets.LIGHTNING_API_KEY }} + API_USERNAME: ${{ secrets.LIGHTNING_API_USERNAME }} + PROJECT_ID: ${{ secrets.LIGHTNING_PROJECT_ID }} + LIGHTNING_MODEL_STORE_TESTING: 1 + run: | + python -m coverage run --source lightning \ + -m pytest -m "not cloud" tests_cloud \ + --timeout=300 -vvvv --durations=50 + + - name: Statistics + if: success() + working-directory: ./tests + run: | + coverage xml -i + coverage report -i + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: tests/coverage.xml + flags: ${COVERAGE_SCOPE},cpu,pytest + env_vars: OS,PYTHON + name: codecov-umbrella + fail_ci_if_error: false From daad2947783c6cd5df1f23457dd6193b92e9fb06 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 20 Jan 2023 14:04:37 +0100 Subject: [PATCH 10/42] azure --- .azure/app-cloud-e2e.yml | 3 +- .azure/app-cloud-store.yml | 63 +++++++++++++++++ .github/workflows/ci-tests-cloud.yml | 101 --------------------------- 3 files changed, 64 insertions(+), 103 deletions(-) create mode 100644 .azure/app-cloud-store.yml delete mode 100644 .github/workflows/ci-tests-cloud.yml diff --git a/.azure/app-cloud-e2e.yml b/.azure/app-cloud-e2e.yml index ff62d9b9d0077..ea085b1ef1c3a 100644 --- a/.azure/app-cloud-e2e.yml +++ b/.azure/app-cloud-e2e.yml @@ -116,6 +116,7 @@ jobs: workspace: clean: all variables: + FREEZE_REQUIREMENTS: "1" HEADLESS: '1' PACKAGE_LIGHTNING: '1' CLOUD: '1' @@ -145,8 +146,6 @@ jobs: - bash: | pip install -e .[test] \ -f https://download.pytorch.org/whl/cpu/torch_stable.html - env: - FREEZE_REQUIREMENTS: "1" displayName: 'Install Lightning & dependencies' - bash: | diff --git a/.azure/app-cloud-store.yml b/.azure/app-cloud-store.yml new file mode 100644 index 0000000000000..ca3e478c9fc5e --- /dev/null +++ b/.azure/app-cloud-store.yml @@ -0,0 +1,63 @@ +# Python package +# Create and test a Python package on multiple Python versions. +# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/python + +trigger: + tags: + include: + - '*' + branches: + include: + - "master" + - "release/*" + - "refs/tags/*" + +pr: + branches: + include: + - "master" + - "release/*" + paths: + include: + - ".actions/**" + - ".azure/app-cloud-store.yml" + - "src/lightning/store/**" + - "tests/tests_cloud/**" + - "setup.py" + exclude: + - "*.md" + - "**/*.md" + +jobs: + - job: test_store + pool: + vmImage: $(imageName) + strategy: + matrix: + Linux: + imageName: 'ubuntu-20.04' + Mac: + imageName: 'macOS-10.14' + Windows: + imageName: 'windows-2019' + timeoutInMinutes: "20" + cancelTimeoutInMinutes: "1" + workspace: + clean: all + variables: + FREEZE_REQUIREMENTS: "1" + LIGHTNING_MODEL_STORE_TESTING: "1" + TORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html" + steps: + + - bash: pip install -e .[test] -f $(TORCH_URL) + displayName: 'Install Lightning & dependencies' + + - bash: | + python -m pytest -m "not cloud" tests_cloud --timeout=300 -v + env: + API_KEY: $(LIGHTNING_API_KEY_PROD) + API_USERNAME: $(LIGHTNING_USERNAME_PROD) + PROJECT_ID: $(LIGHTNING_PROJECT_ID_PROD) + displayName: 'Run the tests' diff --git a/.github/workflows/ci-tests-cloud.yml b/.github/workflows/ci-tests-cloud.yml deleted file mode 100644 index 3bfb26c1d07a2..0000000000000 --- a/.github/workflows/ci-tests-cloud.yml +++ /dev/null @@ -1,101 +0,0 @@ -name: Test Cloud - -# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows -on: - push: - branches: [master, "release/*"] - pull_request: - branches: [master, "release/*"] - types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped - paths: - - ".actions/**" - - ".github/workflows/ci-tests-cloud.yml" - - "src/lightning/store/**" - - "tests/tests_cloud/**" - - "setup.py" - - "!*.md" - - "!**/*.md" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} - -defaults: - run: - shell: bash - -jobs: - app-pytest: - if: github.event.pull_request.draft == false - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-20.04, macOS-11, windows-2022] - python-version: ["3.8"] - requires: ["oldest", "latest"] - - # Timeout: https://stackoverflow.com/a/59076067/4521646 - timeout-minutes: 30 - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: basic setup - run: pip install -q -r .actions/requirements.txt - - - name: Set min. dependencies - if: ${{ matrix.requires == 'oldest' }} - run: python .actions/assistant.py replace_oldest_ver - - - name: Get pip cache dir - id: pip-cache - run: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - - name: Cache pip - uses: actions/cache@v3 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ hashFiles('requirements/app/base.txt') }} - restore-keys: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.requires }}- - - - name: Install package & dependencies - env: - FREEZE_REQUIREMENTS: 1 - TORCH_URL: https://download.pytorch.org/whl/cpu/torch_stable.html - run: | - pip install -e .[dev] --upgrade --find-links ${TORCH_URL} - pip list - - - name: Tests - working-directory: ./tests - env: - API_KEY: ${{ secrets.LIGHTNING_API_KEY }} - API_USERNAME: ${{ secrets.LIGHTNING_API_USERNAME }} - PROJECT_ID: ${{ secrets.LIGHTNING_PROJECT_ID }} - LIGHTNING_MODEL_STORE_TESTING: 1 - run: | - python -m coverage run --source lightning \ - -m pytest -m "not cloud" tests_cloud \ - --timeout=300 -vvvv --durations=50 - - - name: Statistics - if: success() - working-directory: ./tests - run: | - coverage xml -i - coverage report -i - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: tests/coverage.xml - flags: ${COVERAGE_SCOPE},cpu,pytest - env_vars: OS,PYTHON - name: codecov-umbrella - fail_ci_if_error: false From 8dec3950bc764693a5c4c234f67c361589bcf5d8 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 20 Jan 2023 14:16:29 +0100 Subject: [PATCH 11/42] latest --- .azure/app-cloud-store.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure/app-cloud-store.yml b/.azure/app-cloud-store.yml index ca3e478c9fc5e..f045c1bb25139 100644 --- a/.azure/app-cloud-store.yml +++ b/.azure/app-cloud-store.yml @@ -36,11 +36,11 @@ jobs: strategy: matrix: Linux: - imageName: 'ubuntu-20.04' + imageName: 'ubuntu-latest' Mac: - imageName: 'macOS-10.14' + imageName: 'macOS-latest' Windows: - imageName: 'windows-2019' + imageName: 'windows-latest' timeoutInMinutes: "20" cancelTimeoutInMinutes: "1" workspace: From 0e22d6c736c1208b6affed259ae18a7efd63e86b Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 20 Jan 2023 14:30:30 +0100 Subject: [PATCH 12/42] py3.9 --- .azure/app-cloud-store.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.azure/app-cloud-store.yml b/.azure/app-cloud-store.yml index f045c1bb25139..500c8619eae56 100644 --- a/.azure/app-cloud-store.yml +++ b/.azure/app-cloud-store.yml @@ -50,6 +50,9 @@ jobs: LIGHTNING_MODEL_STORE_TESTING: "1" TORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html" steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.9' - bash: pip install -e .[test] -f $(TORCH_URL) displayName: 'Install Lightning & dependencies' From 112c6b94bf6edc4de0ab70f65cc45a441cc2bb85 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 20 Jan 2023 14:34:48 +0100 Subject: [PATCH 13/42] dir --- .azure/app-cloud-store.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure/app-cloud-store.yml b/.azure/app-cloud-store.yml index 500c8619eae56..4ae17af5fb889 100644 --- a/.azure/app-cloud-store.yml +++ b/.azure/app-cloud-store.yml @@ -59,6 +59,7 @@ jobs: - bash: | python -m pytest -m "not cloud" tests_cloud --timeout=300 -v + workingDirectory: tests/ env: API_KEY: $(LIGHTNING_API_KEY_PROD) API_USERNAME: $(LIGHTNING_USERNAME_PROD) From e1268f3290965d3bb01cfa744453c319953f666d Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 20 Jan 2023 15:22:55 +0100 Subject: [PATCH 14/42] _ --- .azure/app-cloud-store.yml | 3 ++- src/lightning/store/authentication.py | 4 ++-- src/lightning/store/cloud_api.py | 12 ++++++------ src/lightning/store/save.py | 4 ++-- src/lightning/store/utils.py | 14 +++++++------- tests/tests_cloud/test_model.py | 10 +++++----- tests/tests_cloud/test_requirements.py | 6 +++--- tests/tests_cloud/test_versioning.py | 4 ++-- 8 files changed, 29 insertions(+), 28 deletions(-) diff --git a/.azure/app-cloud-store.yml b/.azure/app-cloud-store.yml index 4ae17af5fb889..f9a5ab8fd4a5a 100644 --- a/.azure/app-cloud-store.yml +++ b/.azure/app-cloud-store.yml @@ -63,5 +63,6 @@ jobs: env: API_KEY: $(LIGHTNING_API_KEY_PROD) API_USERNAME: $(LIGHTNING_USERNAME_PROD) - PROJECT_ID: $(LIGHTNING_PROJECT_ID_PROD) + PROJECT_ID: $(LIGHTNING_PROJECT_ID_STORE) + LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD) displayName: 'Run the tests' diff --git a/src/lightning/store/authentication.py b/src/lightning/store/authentication.py index aa2a78ba460b8..e0a102f28590e 100644 --- a/src/lightning/store/authentication.py +++ b/src/lightning/store/authentication.py @@ -19,7 +19,7 @@ from requests.models import HTTPBasicAuth from lightning.app.utilities.network import LightningClient -from lightning.store.utils import LIGHTNING_CLOUD_URL +from lightning.store.utils import _LIGHTNING_CLOUD_URL def get_user_details(): @@ -34,7 +34,7 @@ def _get_user_details(): def get_username_from_api_key(api_key: str): response = requests.get( - url=f"{LIGHTNING_CLOUD_URL}/v1/auth/user", + url=f"{_LIGHTNING_CLOUD_URL}/v1/auth/user", auth=HTTPBasicAuth("lightning", api_key), ) assert response.status_code == 200, ( diff --git a/src/lightning/store/cloud_api.py b/src/lightning/store/cloud_api.py index fa2b72b192bd5..95ba7194fbab1 100644 --- a/src/lightning/store/cloud_api.py +++ b/src/lightning/store/cloud_api.py @@ -35,12 +35,12 @@ _write_and_save_requirements, get_linked_output_dir, ) -from lightning.store.utils import get_model_data, LIGHTNING_CLOUD_URL, LIGHTNING_STORAGE_FILE, split_name, stage +from lightning.store.utils import get_model_data, _LIGHTNING_CLOUD_URL, _LIGHTNING_STORAGE_FILE, split_name, stage if os.getenv("LIGHTNING_MODEL_STORE_TESTING", 0): from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR else: - from lightning.store.utils import LIGHTNING_STORAGE_DIR + from lightning.store.utils import _LIGHTNING_STORAGE_DIR import lightning as L import pytorch_lightning as PL @@ -262,7 +262,7 @@ def download_from_lightning_cloud( linked_output_dir = "" if not output_dir: - output_dir = LIGHTNING_STORAGE_DIR + output_dir = _LIGHTNING_STORAGE_DIR output_dir = os.path.join(output_dir, username, model_name, version) linked_output_dir = get_linked_output_dir(output_dir) else: @@ -271,7 +271,7 @@ def download_from_lightning_cloud( if not os.path.isdir(output_dir): os.makedirs(output_dir) - response = requests.get(f"{LIGHTNING_CLOUD_URL}/v1/models?name={username}/{model_name}&version={version}") + response = requests.get(f"{_LIGHTNING_CLOUD_URL}/v1/models?name={username}/{model_name}&version={version}") assert response.status_code == 200, ( f"Unable to download the model with name {name} and version {version}." " Maybe reach out to the model owner or check the arguments again?" @@ -294,7 +294,7 @@ def download_from_lightning_cloud( os.symlink(output_dir, linked_output_dir) - with open(LIGHTNING_STORAGE_FILE, "w+") as storage_file: + with open(_LIGHTNING_STORAGE_FILE, "w+") as storage_file: storage = { username: { model_name: { @@ -354,7 +354,7 @@ def load_from_lightning_cloud( " it's expected that only one of them are requested in a single call." ) - if os.path.exists(LIGHTNING_STORAGE_FILE): + if os.path.exists(_LIGHTNING_STORAGE_FILE): version = version or "latest" model_data = get_model_data(name, version) output_dir = model_data["output_dir"] diff --git a/src/lightning/store/save.py b/src/lightning/store/save.py index a50ddaa1e4de1..f81cfdf515746 100644 --- a/src/lightning/store/save.py +++ b/src/lightning/store/save.py @@ -26,7 +26,7 @@ from tqdm import tqdm from tqdm.utils import CallbackIOWrapper -from lightning.store.utils import LIGHTNING_CLOUD_URL +from lightning.store.utils import _LIGHTNING_CLOUD_URL logging.basicConfig(level=logging.INFO) @@ -162,7 +162,7 @@ def _get_url(response_content): if project_id: json_field["project_id"] = project_id response = requests.post( - f"{LIGHTNING_CLOUD_URL}/v1/models", + f"{_LIGHTNING_CLOUD_URL}/v1/models", auth=HTTPBasicAuth(username, api_key), json=json_field, ) diff --git a/src/lightning/store/utils.py b/src/lightning/store/utils.py index 99395323e5806..ebfeac35a38a4 100644 --- a/src/lightning/store/utils.py +++ b/src/lightning/store/utils.py @@ -23,10 +23,10 @@ class stage(Enum): DOWNLOAD = 2 -LIGHTNING_DIR = os.path.join(os.path.expanduser("~"), ".lightning") -LIGHTNING_STORAGE_FILE = os.path.join(LIGHTNING_DIR, ".lightning_model_storage") -LIGHTNING_STORAGE_DIR = os.path.join(LIGHTNING_DIR, "lightning_model_store") -LIGHTNING_CLOUD_URL = os.getenv("LIGHTNING_CLOUD_URL", default="https://lightning.ai") +_LIGHTNING_DIR = os.path.join(os.path.expanduser("~"), ".lightning") +_LIGHTNING_STORAGE_FILE = os.path.join(_LIGHTNING_DIR, ".lightning_model_storage") +_LIGHTNING_STORAGE_DIR = os.path.join(_LIGHTNING_DIR, "lightning_model_store") +_LIGHTNING_CLOUD_URL = os.getenv("LIGHTNING_CLOUD_URL", default="https://lightning.ai") def _check_version(version: str): @@ -59,10 +59,10 @@ def get_model_data(name: str, version: str): username, model_name, version = split_name(name, version, stage.LOAD) assert os.path.exists( - LIGHTNING_STORAGE_FILE - ), f"ERROR: Could not find {LIGHTNING_STORAGE_FILE} (to be generated after download_from_lightning_cloud(...))" + _LIGHTNING_STORAGE_FILE + ), f"ERROR: Could not find {_LIGHTNING_STORAGE_FILE} (to be generated after download_from_lightning_cloud(...))" - with open(LIGHTNING_STORAGE_FILE) as storage_file: + with open(_LIGHTNING_STORAGE_FILE) as storage_file: storage_data = json.load(storage_file) assert username in storage_data, ( diff --git a/tests/tests_cloud/test_model.py b/tests/tests_cloud/test_model.py index a3a8fcbc2117d..8db27d130d4f6 100644 --- a/tests/tests_cloud/test_model.py +++ b/tests/tests_cloud/test_model.py @@ -9,7 +9,7 @@ if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR else: - from lightning.store.utils import LIGHTNING_STORAGE_DIR + from lightning.store.utils import _LIGHTNING_STORAGE_DIR def test_model(): @@ -31,7 +31,7 @@ def test_model(): ) download_from_lightning_cloud(f"{username}/{model_name}") - assert os.path.isdir(os.path.join(LIGHTNING_STORAGE_DIR, username, model_name, version)) + assert os.path.isdir(os.path.join(_LIGHTNING_STORAGE_DIR, username, model_name, version)) model = load_from_lightning_cloud(f"{username}/{model_name}") assert model is not None @@ -57,7 +57,7 @@ def test_model_without_progress_bar(): ) download_from_lightning_cloud(f"{username}/{model_name}", progress_bar=False) - assert os.path.isdir(os.path.join(LIGHTNING_STORAGE_DIR, username, model_name, version)) + assert os.path.isdir(os.path.join(_LIGHTNING_STORAGE_DIR, username, model_name, version)) model = load_from_lightning_cloud(f"{username}/{model_name}") assert model is not None @@ -86,7 +86,7 @@ def test_only_weights(): ) download_from_lightning_cloud(f"{username}/{model_name}") - assert os.path.isdir(os.path.join(LIGHTNING_STORAGE_DIR, username, model_name, version)) + assert os.path.isdir(os.path.join(_LIGHTNING_STORAGE_DIR, username, model_name, version)) model_with_weights = load_from_lightning_cloud(f"{username}/{model_name}", load_weights=True, model=model) assert model_with_weights is not None @@ -116,7 +116,7 @@ def test_checkpoint_path(): ) download_from_lightning_cloud(f"{username}/{model_name}") - assert os.path.isdir(os.path.join(LIGHTNING_STORAGE_DIR, username, model_name, version)) + assert os.path.isdir(os.path.join(_LIGHTNING_STORAGE_DIR, username, model_name, version)) ckpt = load_from_lightning_cloud(f"{username}/{model_name}", load_checkpoint=True, model=model) assert ckpt is not None diff --git a/tests/tests_cloud/test_requirements.py b/tests/tests_cloud/test_requirements.py index 571e4f5037705..fcc8ec031f08a 100644 --- a/tests/tests_cloud/test_requirements.py +++ b/tests/tests_cloud/test_requirements.py @@ -8,7 +8,7 @@ if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR else: - from lightning.store.utils import LIGHTNING_STORAGE_DIR + from lightning.store.utils import _LIGHTNING_STORAGE_DIR def test_requirements_as_a_file(): @@ -34,7 +34,7 @@ def test_requirements_as_a_file(): download_from_lightning_cloud(f"{username}/{model_name}") - req_folder_path = os.path.join(LIGHTNING_STORAGE_DIR, username, model_name, version) + req_folder_path = os.path.join(_LIGHTNING_STORAGE_DIR, username, model_name, version) assert os.path.isdir(req_folder_path) assert "requirements.txt" in os.listdir(req_folder_path) @@ -62,7 +62,7 @@ def test_requirements_as_a_list(): download_from_lightning_cloud(f"{username}/{model_name}", version=version) - req_folder_path = os.path.join(LIGHTNING_STORAGE_DIR, username, model_name, version) + req_folder_path = os.path.join(_LIGHTNING_STORAGE_DIR, username, model_name, version) assert os.path.isdir(req_folder_path) assert "requirements.txt" in os.listdir(req_folder_path) diff --git a/tests/tests_cloud/test_versioning.py b/tests/tests_cloud/test_versioning.py index 132f913068ad6..185c061068df4 100644 --- a/tests/tests_cloud/test_versioning.py +++ b/tests/tests_cloud/test_versioning.py @@ -10,12 +10,12 @@ if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR else: - from lightning.store.utils import LIGHTNING_STORAGE_DIR + from lightning.store.utils import _LIGHTNING_STORAGE_DIR def assert_download_successful(username, model_name, version): folder_name = os.path.join( - LIGHTNING_STORAGE_DIR, + _LIGHTNING_STORAGE_DIR, username, model_name, version, From b910df4dcb0b965c122ea49cb7228afe69983dc0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 20 Jan 2023 14:23:58 +0000 Subject: [PATCH 15/42] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/lightning/store/cloud_api.py | 4 ++-- tests/tests_cloud/test_model.py | 2 +- tests/tests_cloud/test_requirements.py | 2 +- tests/tests_cloud/test_versioning.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lightning/store/cloud_api.py b/src/lightning/store/cloud_api.py index 95ba7194fbab1..4fa1779250bb0 100644 --- a/src/lightning/store/cloud_api.py +++ b/src/lightning/store/cloud_api.py @@ -35,10 +35,10 @@ _write_and_save_requirements, get_linked_output_dir, ) -from lightning.store.utils import get_model_data, _LIGHTNING_CLOUD_URL, _LIGHTNING_STORAGE_FILE, split_name, stage +from lightning.store.utils import _LIGHTNING_CLOUD_URL, _LIGHTNING_STORAGE_FILE, get_model_data, split_name, stage if os.getenv("LIGHTNING_MODEL_STORE_TESTING", 0): - from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR + pass else: from lightning.store.utils import _LIGHTNING_STORAGE_DIR diff --git a/tests/tests_cloud/test_model.py b/tests/tests_cloud/test_model.py index 8db27d130d4f6..40532bc424c46 100644 --- a/tests/tests_cloud/test_model.py +++ b/tests/tests_cloud/test_model.py @@ -7,7 +7,7 @@ from pytorch_lightning.demos.boring_classes import BoringModel if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): - from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR + pass else: from lightning.store.utils import _LIGHTNING_STORAGE_DIR diff --git a/tests/tests_cloud/test_requirements.py b/tests/tests_cloud/test_requirements.py index fcc8ec031f08a..8823f58567abe 100644 --- a/tests/tests_cloud/test_requirements.py +++ b/tests/tests_cloud/test_requirements.py @@ -6,7 +6,7 @@ from pytorch_lightning.demos.boring_classes import BoringModel if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): - from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR + pass else: from lightning.store.utils import _LIGHTNING_STORAGE_DIR diff --git a/tests/tests_cloud/test_versioning.py b/tests/tests_cloud/test_versioning.py index 185c061068df4..2f1e342de776a 100644 --- a/tests/tests_cloud/test_versioning.py +++ b/tests/tests_cloud/test_versioning.py @@ -8,7 +8,7 @@ from pytorch_lightning.demos.boring_classes import BoringModel if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): - from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR + pass else: from lightning.store.utils import _LIGHTNING_STORAGE_DIR From 03781df7f4213d6d27aff592edc60f2e9f3f3055 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 20 Jan 2023 15:39:02 +0100 Subject: [PATCH 16/42] tests --- src/lightning/store/cloud_api.py | 4 ++-- tests/tests_cloud/__init__.py | 10 ++++++++-- tests/tests_cloud/test_model.py | 14 +++++--------- tests/tests_cloud/test_requirements.py | 10 +++------- tests/tests_cloud/test_source_code.py | 16 ++++++---------- tests/tests_cloud/test_versioning.py | 8 ++------ tests/tests_cloud/utils.py | 5 ++--- 7 files changed, 28 insertions(+), 39 deletions(-) diff --git a/src/lightning/store/cloud_api.py b/src/lightning/store/cloud_api.py index 95ba7194fbab1..06005b1087abd 100644 --- a/src/lightning/store/cloud_api.py +++ b/src/lightning/store/cloud_api.py @@ -35,10 +35,10 @@ _write_and_save_requirements, get_linked_output_dir, ) -from lightning.store.utils import get_model_data, _LIGHTNING_CLOUD_URL, _LIGHTNING_STORAGE_FILE, split_name, stage +from lightning.store.utils import _LIGHTNING_CLOUD_URL, _LIGHTNING_STORAGE_FILE, get_model_data, split_name, stage if os.getenv("LIGHTNING_MODEL_STORE_TESTING", 0): - from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR + from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as _LIGHTNING_STORAGE_DIR else: from lightning.store.utils import _LIGHTNING_STORAGE_DIR diff --git a/tests/tests_cloud/__init__.py b/tests/tests_cloud/__init__.py index be8e23f466544..8c458882f8e02 100644 --- a/tests/tests_cloud/__init__.py +++ b/tests/tests_cloud/__init__.py @@ -1,4 +1,10 @@ import os -LIGHTNING_DIR = f"{os.path.expanduser('~')}/.lightning" -LIGHTNING_TEST_STORAGE_DIR = f"{LIGHTNING_DIR}/lightning_test_model_store/" +_LIGHTNING_DIR = f"{os.path.expanduser('~')}/.lightning" + +if os.getenv("LIGHTNING_MODEL_STORE_TESTING") == "1": + STORAGE_DIR = f"{_LIGHTNING_DIR}/lightning_test_model_store/" +else: + from lightning.store.utils import _LIGHTNING_STORAGE_DIR as STORAGE_DIR + +__all__ = ["STORAGE_DIR"] diff --git a/tests/tests_cloud/test_model.py b/tests/tests_cloud/test_model.py index 8db27d130d4f6..a9e8c9d1f9ae6 100644 --- a/tests/tests_cloud/test_model.py +++ b/tests/tests_cloud/test_model.py @@ -1,16 +1,12 @@ import os +from tests_cloud import STORAGE_DIR from tests_cloud.utils import cleanup import pytorch_lightning as pl from lightning.store import download_from_lightning_cloud, load_from_lightning_cloud, to_lightning_cloud from pytorch_lightning.demos.boring_classes import BoringModel -if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): - from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR -else: - from lightning.store.utils import _LIGHTNING_STORAGE_DIR - def test_model(): cleanup() @@ -31,7 +27,7 @@ def test_model(): ) download_from_lightning_cloud(f"{username}/{model_name}") - assert os.path.isdir(os.path.join(_LIGHTNING_STORAGE_DIR, username, model_name, version)) + assert os.path.isdir(os.path.join(STORAGE_DIR, username, model_name, version)) model = load_from_lightning_cloud(f"{username}/{model_name}") assert model is not None @@ -57,7 +53,7 @@ def test_model_without_progress_bar(): ) download_from_lightning_cloud(f"{username}/{model_name}", progress_bar=False) - assert os.path.isdir(os.path.join(_LIGHTNING_STORAGE_DIR, username, model_name, version)) + assert os.path.isdir(os.path.join(STORAGE_DIR, username, model_name, version)) model = load_from_lightning_cloud(f"{username}/{model_name}") assert model is not None @@ -86,7 +82,7 @@ def test_only_weights(): ) download_from_lightning_cloud(f"{username}/{model_name}") - assert os.path.isdir(os.path.join(_LIGHTNING_STORAGE_DIR, username, model_name, version)) + assert os.path.isdir(os.path.join(STORAGE_DIR, username, model_name, version)) model_with_weights = load_from_lightning_cloud(f"{username}/{model_name}", load_weights=True, model=model) assert model_with_weights is not None @@ -116,7 +112,7 @@ def test_checkpoint_path(): ) download_from_lightning_cloud(f"{username}/{model_name}") - assert os.path.isdir(os.path.join(_LIGHTNING_STORAGE_DIR, username, model_name, version)) + assert os.path.isdir(os.path.join(STORAGE_DIR, username, model_name, version)) ckpt = load_from_lightning_cloud(f"{username}/{model_name}", load_checkpoint=True, model=model) assert ckpt is not None diff --git a/tests/tests_cloud/test_requirements.py b/tests/tests_cloud/test_requirements.py index fcc8ec031f08a..14b5c53131e1c 100644 --- a/tests/tests_cloud/test_requirements.py +++ b/tests/tests_cloud/test_requirements.py @@ -1,15 +1,11 @@ import os +from tests_cloud import STORAGE_DIR from tests_cloud.utils import cleanup from lightning.store import download_from_lightning_cloud, to_lightning_cloud from pytorch_lightning.demos.boring_classes import BoringModel -if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): - from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR -else: - from lightning.store.utils import _LIGHTNING_STORAGE_DIR - def test_requirements_as_a_file(): cleanup() @@ -34,7 +30,7 @@ def test_requirements_as_a_file(): download_from_lightning_cloud(f"{username}/{model_name}") - req_folder_path = os.path.join(_LIGHTNING_STORAGE_DIR, username, model_name, version) + req_folder_path = os.path.join(STORAGE_DIR, username, model_name, version) assert os.path.isdir(req_folder_path) assert "requirements.txt" in os.listdir(req_folder_path) @@ -62,7 +58,7 @@ def test_requirements_as_a_list(): download_from_lightning_cloud(f"{username}/{model_name}", version=version) - req_folder_path = os.path.join(_LIGHTNING_STORAGE_DIR, username, model_name, version) + req_folder_path = os.path.join(STORAGE_DIR, username, model_name, version) assert os.path.isdir(req_folder_path) assert "requirements.txt" in os.listdir(req_folder_path) diff --git a/tests/tests_cloud/test_source_code.py b/tests/tests_cloud/test_source_code.py index 68da410a5ddb9..ae8a17741cb49 100644 --- a/tests/tests_cloud/test_source_code.py +++ b/tests/tests_cloud/test_source_code.py @@ -2,16 +2,12 @@ import os import tempfile +from tests_cloud import STORAGE_DIR from tests_cloud.utils import cleanup from lightning.store import download_from_lightning_cloud, to_lightning_cloud from pytorch_lightning.demos.boring_classes import BoringModel -if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): - from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR -else: - from lightning.store import LIGHTNING_STORAGE_DIR - def test_source_code_implicit(): cleanup() @@ -33,7 +29,7 @@ def test_source_code_implicit(): download_from_lightning_cloud(f"{username}/{model_name}") assert os.path.isfile( os.path.join( - LIGHTNING_STORAGE_DIR, + STORAGE_DIR, username, model_name, "latest", @@ -62,7 +58,7 @@ def test_source_code_saving_disabled(): download_from_lightning_cloud(f"{username}/{model_name}") assert not os.path.isfile( os.path.join( - LIGHTNING_STORAGE_DIR, + STORAGE_DIR, username, model_name, "latest", @@ -94,7 +90,7 @@ def test_source_code_explicit_relative_folder(): assert os.path.isdir( os.path.join( - LIGHTNING_STORAGE_DIR, + STORAGE_DIR, username, model_name, "latest", @@ -127,7 +123,7 @@ def test_source_code_explicit_absolute_folder(): assert os.path.isdir( os.path.join( - LIGHTNING_STORAGE_DIR, + STORAGE_DIR, username, model_name, "latest", @@ -159,7 +155,7 @@ def test_source_code_explicit_file(): assert os.path.isfile( os.path.join( - LIGHTNING_STORAGE_DIR, + STORAGE_DIR, username, model_name, "latest", diff --git a/tests/tests_cloud/test_versioning.py b/tests/tests_cloud/test_versioning.py index 185c061068df4..a5f8f94e0d305 100644 --- a/tests/tests_cloud/test_versioning.py +++ b/tests/tests_cloud/test_versioning.py @@ -2,20 +2,16 @@ import platform import pytest +from tests_cloud import STORAGE_DIR from tests_cloud.utils import cleanup from lightning.store.cloud_api import download_from_lightning_cloud, to_lightning_cloud from pytorch_lightning.demos.boring_classes import BoringModel -if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): - from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as LIGHTNING_STORAGE_DIR -else: - from lightning.store.utils import _LIGHTNING_STORAGE_DIR - def assert_download_successful(username, model_name, version): folder_name = os.path.join( - _LIGHTNING_STORAGE_DIR, + STORAGE_DIR, username, model_name, version, diff --git a/tests/tests_cloud/utils.py b/tests/tests_cloud/utils.py index 8fd46e1b23740..f96e8e1838c2c 100644 --- a/tests/tests_cloud/utils.py +++ b/tests/tests_cloud/utils.py @@ -5,6 +5,5 @@ def cleanup(): - if os.getenv("LIGHTNING_MODEL_STORE_TESTING"): - if os.path.isdir(LIGHTNING_TEST_STORAGE_DIR): - shutil.rmtree(LIGHTNING_TEST_STORAGE_DIR) + if os.getenv("LIGHTNING_MODEL_STORE_TESTING") and os.path.isdir(LIGHTNING_TEST_STORAGE_DIR): + shutil.rmtree(LIGHTNING_TEST_STORAGE_DIR) From 86d5820602772e81b67791fe80bb99583006be58 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 20 Jan 2023 15:45:58 +0100 Subject: [PATCH 17/42] imports --- tests/tests_cloud/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/tests_cloud/utils.py b/tests/tests_cloud/utils.py index f96e8e1838c2c..69280240109a2 100644 --- a/tests/tests_cloud/utils.py +++ b/tests/tests_cloud/utils.py @@ -1,9 +1,9 @@ import os import shutil -from tests_cloud import LIGHTNING_TEST_STORAGE_DIR +from tests_cloud import STORAGE_DIR def cleanup(): - if os.getenv("LIGHTNING_MODEL_STORE_TESTING") and os.path.isdir(LIGHTNING_TEST_STORAGE_DIR): - shutil.rmtree(LIGHTNING_TEST_STORAGE_DIR) + if os.getenv("LIGHTNING_MODEL_STORE_TESTING") and os.path.isdir(STORAGE_DIR): + shutil.rmtree(STORAGE_DIR) From 4938a8d29360c39c4091e720550ee9f27da8b63e Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 20 Jan 2023 16:03:27 +0100 Subject: [PATCH 18/42] imports --- src/lightning/store/cloud_api.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/lightning/store/cloud_api.py b/src/lightning/store/cloud_api.py index 06005b1087abd..6bf6802b90170 100644 --- a/src/lightning/store/cloud_api.py +++ b/src/lightning/store/cloud_api.py @@ -22,6 +22,8 @@ import requests import torch +import lightning as L +import pytorch_lightning as PL from lightning.store.authentication import authenticate from lightning.store.save import ( _download_and_extract_data_to, @@ -35,15 +37,14 @@ _write_and_save_requirements, get_linked_output_dir, ) -from lightning.store.utils import _LIGHTNING_CLOUD_URL, _LIGHTNING_STORAGE_FILE, get_model_data, split_name, stage - -if os.getenv("LIGHTNING_MODEL_STORE_TESTING", 0): - from tests_cloud import LIGHTNING_TEST_STORAGE_DIR as _LIGHTNING_STORAGE_DIR -else: - from lightning.store.utils import _LIGHTNING_STORAGE_DIR - -import lightning as L -import pytorch_lightning as PL +from lightning.store.utils import ( + _LIGHTNING_CLOUD_URL, + _LIGHTNING_STORAGE_DIR, + _LIGHTNING_STORAGE_FILE, + get_model_data, + split_name, + stage, +) logging.basicConfig(level=logging.INFO) From 29e41d16c17e75d5c082976f2c5ae71165ebe547 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 20 Jan 2023 17:25:45 +0100 Subject: [PATCH 19/42] cleaning --- tests/tests_cloud/__init__.py | 6 ++ tests/tests_cloud/{utils.py => helpers.py} | 0 tests/tests_cloud/test_model.py | 68 ++++++---------------- tests/tests_cloud/test_requirements.py | 32 +++------- tests/tests_cloud/test_source_code.py | 68 ++++++---------------- tests/tests_cloud/test_versioning.py | 33 +++-------- 6 files changed, 58 insertions(+), 149 deletions(-) rename tests/tests_cloud/{utils.py => helpers.py} (100%) diff --git a/tests/tests_cloud/__init__.py b/tests/tests_cloud/__init__.py index 8c458882f8e02..f49d549afd0ab 100644 --- a/tests/tests_cloud/__init__.py +++ b/tests/tests_cloud/__init__.py @@ -7,4 +7,10 @@ else: from lightning.store.utils import _LIGHTNING_STORAGE_DIR as STORAGE_DIR +_USERNAME = os.getenv("API_USERNAME", "") +if not _USERNAME: + raise ValueError( + "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" + ) + __all__ = ["STORAGE_DIR"] diff --git a/tests/tests_cloud/utils.py b/tests/tests_cloud/helpers.py similarity index 100% rename from tests/tests_cloud/utils.py rename to tests/tests_cloud/helpers.py diff --git a/tests/tests_cloud/test_model.py b/tests/tests_cloud/test_model.py index a9e8c9d1f9ae6..48764c9df267e 100644 --- a/tests/tests_cloud/test_model.py +++ b/tests/tests_cloud/test_model.py @@ -1,23 +1,15 @@ import os -from tests_cloud import STORAGE_DIR -from tests_cloud.utils import cleanup +from tests_cloud import _USERNAME, STORAGE_DIR +from tests_cloud.helpers import cleanup import pytorch_lightning as pl from lightning.store import download_from_lightning_cloud, load_from_lightning_cloud, to_lightning_cloud from pytorch_lightning.demos.boring_classes import BoringModel -def test_model(): +def test_model(model_name: str = "boring_model", version: str = "latest"): cleanup() - username = os.getenv("API_USERNAME", "") - if not username: - raise ValueError( - "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" - ) - - model_name = "boring_model" - version = "latest" to_lightning_cloud( model_name, @@ -26,23 +18,15 @@ def test_model(): project_id=os.getenv("PROJECT_ID", ""), ) - download_from_lightning_cloud(f"{username}/{model_name}") - assert os.path.isdir(os.path.join(STORAGE_DIR, username, model_name, version)) + download_from_lightning_cloud(f"{_USERNAME}/{model_name}") + assert os.path.isdir(os.path.join(STORAGE_DIR, _USERNAME, model_name, version)) - model = load_from_lightning_cloud(f"{username}/{model_name}") + model = load_from_lightning_cloud(f"{_USERNAME}/{model_name}") assert model is not None -def test_model_without_progress_bar(): +def test_model_without_progress_bar(model_name: str = "boring_model", version: str = "latest"): cleanup() - username = os.getenv("API_USERNAME", "") - if not username: - raise ValueError( - "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" - ) - - model_name = "boring_model" - version = "latest" to_lightning_cloud( model_name, @@ -52,23 +36,15 @@ def test_model_without_progress_bar(): progress_bar=False, ) - download_from_lightning_cloud(f"{username}/{model_name}", progress_bar=False) - assert os.path.isdir(os.path.join(STORAGE_DIR, username, model_name, version)) + download_from_lightning_cloud(f"{_USERNAME}/{model_name}", progress_bar=False) + assert os.path.isdir(os.path.join(STORAGE_DIR, _USERNAME, model_name, version)) - model = load_from_lightning_cloud(f"{username}/{model_name}") + model = load_from_lightning_cloud(f"{_USERNAME}/{model_name}") assert model is not None -def test_only_weights(): +def test_only_weights(model_name: str = "boring_model_only_weights", version: str = "latest"): cleanup() - username = os.getenv("API_USERNAME", "") - if not username: - raise ValueError( - "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" - ) - - model_name = "boring_model_only_weights" - version = "latest" model = BoringModel() trainer = pl.Trainer(fast_dev_run=True) @@ -81,24 +57,16 @@ def test_only_weights(): project_id=os.getenv("PROJECT_ID", ""), ) - download_from_lightning_cloud(f"{username}/{model_name}") - assert os.path.isdir(os.path.join(STORAGE_DIR, username, model_name, version)) + download_from_lightning_cloud(f"{_USERNAME}/{model_name}") + assert os.path.isdir(os.path.join(STORAGE_DIR, _USERNAME, model_name, version)) - model_with_weights = load_from_lightning_cloud(f"{username}/{model_name}", load_weights=True, model=model) + model_with_weights = load_from_lightning_cloud(f"{_USERNAME}/{model_name}", load_weights=True, model=model) assert model_with_weights is not None assert model_with_weights.state_dict() is not None -def test_checkpoint_path(): +def test_checkpoint_path(model_name: str = "boring_model_only_checkpoint_path", version: str = "latest"): cleanup() - username = os.getenv("API_USERNAME", "") - if not username: - raise ValueError( - "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" - ) - - model_name = "boring_model_only_checkpoint_path" - version = "latest" model = BoringModel() trainer = pl.Trainer(fast_dev_run=True) @@ -111,8 +79,8 @@ def test_checkpoint_path(): project_id=os.getenv("PROJECT_ID", ""), ) - download_from_lightning_cloud(f"{username}/{model_name}") - assert os.path.isdir(os.path.join(STORAGE_DIR, username, model_name, version)) + download_from_lightning_cloud(f"{_USERNAME}/{model_name}") + assert os.path.isdir(os.path.join(STORAGE_DIR, _USERNAME, model_name, version)) - ckpt = load_from_lightning_cloud(f"{username}/{model_name}", load_checkpoint=True, model=model) + ckpt = load_from_lightning_cloud(f"{_USERNAME}/{model_name}", load_checkpoint=True, model=model) assert ckpt is not None diff --git a/tests/tests_cloud/test_requirements.py b/tests/tests_cloud/test_requirements.py index 14b5c53131e1c..47ed98959306a 100644 --- a/tests/tests_cloud/test_requirements.py +++ b/tests/tests_cloud/test_requirements.py @@ -1,23 +1,16 @@ import os -from tests_cloud import STORAGE_DIR -from tests_cloud.utils import cleanup +from tests_cloud import _USERNAME, STORAGE_DIR +from tests_cloud.helpers import cleanup from lightning.store import download_from_lightning_cloud, to_lightning_cloud from pytorch_lightning.demos.boring_classes import BoringModel -def test_requirements_as_a_file(): +def test_requirements_as_a_file(version: str = "latest", model_name: str = "boring_model"): cleanup() - username = os.getenv("API_USERNAME", "") - if not username: - raise ValueError( - "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" - ) requirements_file_path = "tests/requirements.txt" - version = "latest" - model_name = "boring_model" to_lightning_cloud( model_name, @@ -28,23 +21,16 @@ def test_requirements_as_a_file(): project_id=os.getenv("PROJECT_ID", ""), ) - download_from_lightning_cloud(f"{username}/{model_name}") + download_from_lightning_cloud(f"{_USERNAME}/{model_name}") - req_folder_path = os.path.join(STORAGE_DIR, username, model_name, version) + req_folder_path = os.path.join(STORAGE_DIR, _USERNAME, model_name, version) assert os.path.isdir(req_folder_path) assert "requirements.txt" in os.listdir(req_folder_path) -def test_requirements_as_a_list(): +def test_requirements_as_a_list(version: str = "1.0.0", model_name: str = "boring_model"): cleanup() - username = os.getenv("API_USERNAME", "") - if not username: - raise ValueError( - "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" - ) - - version = "1.0.0" - model_name = "boring_model" + requirements_list = ["pytorch_lightning==1.7.7", "lightning"] to_lightning_cloud( @@ -56,9 +42,9 @@ def test_requirements_as_a_list(): project_id=os.getenv("PROJECT_ID", ""), ) - download_from_lightning_cloud(f"{username}/{model_name}", version=version) + download_from_lightning_cloud(f"{_USERNAME}/{model_name}", version=version) - req_folder_path = os.path.join(STORAGE_DIR, username, model_name, version) + req_folder_path = os.path.join(STORAGE_DIR, _USERNAME, model_name, version) assert os.path.isdir(req_folder_path) assert "requirements.txt" in os.listdir(req_folder_path) diff --git a/tests/tests_cloud/test_source_code.py b/tests/tests_cloud/test_source_code.py index ae8a17741cb49..db8f864908433 100644 --- a/tests/tests_cloud/test_source_code.py +++ b/tests/tests_cloud/test_source_code.py @@ -2,23 +2,16 @@ import os import tempfile -from tests_cloud import STORAGE_DIR -from tests_cloud.utils import cleanup +from tests_cloud import _USERNAME, STORAGE_DIR +from tests_cloud.helpers import cleanup from lightning.store import download_from_lightning_cloud, to_lightning_cloud from pytorch_lightning.demos.boring_classes import BoringModel -def test_source_code_implicit(): +def test_source_code_implicit(model_name: str = "model_test_source_code_implicit"): cleanup() - username = os.getenv("API_USERNAME", "") - if not username: - raise ValueError( - "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" - ) - username = os.getenv("API_USERNAME") - model_name = "model_test_source_code_implicit" to_lightning_cloud( model_name, model=BoringModel(), @@ -26,11 +19,11 @@ def test_source_code_implicit(): project_id=os.getenv("PROJECT_ID", ""), ) - download_from_lightning_cloud(f"{username}/{model_name}") + download_from_lightning_cloud(f"{_USERNAME}/{model_name}") assert os.path.isfile( os.path.join( STORAGE_DIR, - username, + _USERNAME, model_name, "latest", str(os.path.basename(inspect.getsourcefile(BoringModel))), @@ -38,15 +31,9 @@ def test_source_code_implicit(): ) -def test_source_code_saving_disabled(): +def test_source_code_saving_disabled(model_name: str = "model_test_source_code_dont_save"): cleanup() - username = os.getenv("API_USERNAME", "") - if not username: - raise ValueError( - "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" - ) - model_name = "model_test_source_code_dont_save" to_lightning_cloud( model_name, model=BoringModel(), @@ -55,11 +42,11 @@ def test_source_code_saving_disabled(): save_code=False, ) - download_from_lightning_cloud(f"{username}/{model_name}") + download_from_lightning_cloud(f"{_USERNAME}/{model_name}") assert not os.path.isfile( os.path.join( STORAGE_DIR, - username, + _USERNAME, model_name, "latest", str(os.path.basename(inspect.getsourcefile(BoringModel))), @@ -67,16 +54,9 @@ def test_source_code_saving_disabled(): ) -def test_source_code_explicit_relative_folder(): +def test_source_code_explicit_relative_folder(model_name: str = "model_test_source_code_explicit_relative"): cleanup() - username = os.getenv("API_USERNAME", "") - if not username: - raise ValueError( - "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" - ) - username = os.getenv("API_USERNAME") - model_name = "model_test_source_code_explicit_relative" dir_upload_path = f"../{os.path.basename(os.getcwd())}/tests/" to_lightning_cloud( model_name, @@ -86,12 +66,12 @@ def test_source_code_explicit_relative_folder(): project_id=os.getenv("PROJECT_ID", ""), ) - download_from_lightning_cloud(f"{username}/{model_name}") + download_from_lightning_cloud(f"{_USERNAME}/{model_name}") assert os.path.isdir( os.path.join( STORAGE_DIR, - username, + _USERNAME, model_name, "latest", os.path.basename(os.path.abspath(dir_upload_path)), @@ -99,16 +79,9 @@ def test_source_code_explicit_relative_folder(): ) -def test_source_code_explicit_absolute_folder(): +def test_source_code_explicit_absolute_folder(model_name: str = "model_test_source_code_explicit_absolute_path"): cleanup() - username = os.getenv("API_USERNAME", "") - if not username: - raise ValueError( - "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" - ) - username = os.getenv("API_USERNAME") - model_name = "model_test_source_code_explicit_absolute_path" with tempfile.TemporaryDirectory() as tmpdir: dir_upload_path = os.path.abspath(tmpdir) to_lightning_cloud( @@ -119,12 +92,12 @@ def test_source_code_explicit_absolute_folder(): project_id=os.getenv("PROJECT_ID", ""), ) - download_from_lightning_cloud(f"{username}/{model_name}") + download_from_lightning_cloud(f"{_USERNAME}/{model_name}") assert os.path.isdir( os.path.join( STORAGE_DIR, - username, + _USERNAME, model_name, "latest", os.path.basename(os.path.abspath(dir_upload_path)), @@ -132,16 +105,9 @@ def test_source_code_explicit_absolute_folder(): ) -def test_source_code_explicit_file(): +def test_source_code_explicit_file(model_name: str = "model_test_source_code_explicit_file"): cleanup() - username = os.getenv("API_USERNAME", "") - if not username: - raise ValueError( - "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" - ) - username = os.getenv("API_USERNAME") - model_name = "model_test_source_code_explicit_file" file_name = os.path.abspath("setup.py") to_lightning_cloud( model_name, @@ -151,12 +117,12 @@ def test_source_code_explicit_file(): project_id=os.getenv("PROJECT_ID", ""), ) - download_from_lightning_cloud(f"{username}/{model_name}") + download_from_lightning_cloud(f"{_USERNAME}/{model_name}") assert os.path.isfile( os.path.join( STORAGE_DIR, - username, + _USERNAME, model_name, "latest", os.path.basename(file_name), diff --git a/tests/tests_cloud/test_versioning.py b/tests/tests_cloud/test_versioning.py index a5f8f94e0d305..30e784a397417 100644 --- a/tests/tests_cloud/test_versioning.py +++ b/tests/tests_cloud/test_versioning.py @@ -2,8 +2,8 @@ import platform import pytest -from tests_cloud import STORAGE_DIR -from tests_cloud.utils import cleanup +from tests_cloud import _USERNAME, STORAGE_DIR +from tests_cloud.helpers import cleanup from lightning.store.cloud_api import download_from_lightning_cloud, to_lightning_cloud from pytorch_lightning.demos.boring_classes import BoringModel @@ -21,10 +21,7 @@ def assert_download_successful(username, model_name, version): @pytest.mark.parametrize( - ( - "case", - "expected_case", - ), + ("case", "expected_case"), ( [ ("1.0.0", "version_1_0_0"), @@ -37,15 +34,8 @@ def assert_download_successful(username, model_name, version): ] ), ) -def test_versioning_valid_case(case, expected_case): +def test_versioning_valid_case(case, expected_case, model_name: str = "boring_model_versioning"): cleanup() - username = os.getenv("API_USERNAME", "") - if not username: - raise ValueError( - "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" - ) - - model_name = "boring_model_versioning" api_key = os.getenv("API_KEY", "") to_lightning_cloud( @@ -55,8 +45,8 @@ def test_versioning_valid_case(case, expected_case): api_key=api_key, project_id=os.getenv("PROJECT_ID", ""), ) - download_from_lightning_cloud(f"{username}/{model_name}", version=case) - assert_download_successful(username, model_name, expected_case) + download_from_lightning_cloud(f"{_USERNAME}/{model_name}", version=case) + assert_download_successful(_USERNAME, model_name, expected_case) @pytest.mark.parametrize( @@ -71,15 +61,8 @@ def test_versioning_valid_case(case, expected_case): ] ), ) -def test_versioning_invalid_case(case): +def test_versioning_invalid_case(case, model_name: str = "boring_model_versioning"): cleanup() - username = os.getenv("API_USERNAME", "") - if not username: - raise ValueError( - "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" - ) - - model_name = "boring_model_versioning" with pytest.raises(AssertionError): api_key = os.getenv("API_KEY", "") @@ -93,4 +76,4 @@ def test_versioning_invalid_case(case): error = OSError if case == "*" and platform.system() == "Windows" else AssertionError with pytest.raises(error): - download_from_lightning_cloud(f"{username}/{model_name}", version=case) + download_from_lightning_cloud(f"{_USERNAME}/{model_name}", version=case) From e157483977b7d61a3170796c574444161ea39da0 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 20 Jan 2023 17:37:54 +0100 Subject: [PATCH 20/42] env --- tests/tests_cloud/__init__.py | 9 ++-- tests/tests_cloud/helpers.py | 1 + tests/tests_cloud/test_model.py | 32 ++----------- tests/tests_cloud/test_requirements.py | 10 ++-- tests/tests_cloud/test_source_code.py | 63 ++++---------------------- tests/tests_cloud/test_versioning.py | 27 ++--------- 6 files changed, 30 insertions(+), 112 deletions(-) diff --git a/tests/tests_cloud/__init__.py b/tests/tests_cloud/__init__.py index f49d549afd0ab..0ed8c529d9f88 100644 --- a/tests/tests_cloud/__init__.py +++ b/tests/tests_cloud/__init__.py @@ -8,9 +8,10 @@ from lightning.store.utils import _LIGHTNING_STORAGE_DIR as STORAGE_DIR _USERNAME = os.getenv("API_USERNAME", "") -if not _USERNAME: - raise ValueError( - "No API_USERNAME env variable, to test, make sure to add export API_USERNAME='yourusername' before testing" - ) +assert _USERNAME, "No API_USERNAME env variable, make sure to add it before testing" +_API_KEY = os.getenv("API_KEY", "") +assert _API_KEY, "No API_KEY env variable, make sure to add it before testing" +_PROJECT_ID = os.getenv("PROJECT_ID", "") +assert _PROJECT_ID, "No PROJECT_ID env variable, make sure to add it before testing" __all__ = ["STORAGE_DIR"] diff --git a/tests/tests_cloud/helpers.py b/tests/tests_cloud/helpers.py index 69280240109a2..7af0dda1bd31b 100644 --- a/tests/tests_cloud/helpers.py +++ b/tests/tests_cloud/helpers.py @@ -4,6 +4,7 @@ from tests_cloud import STORAGE_DIR +# TODO: make this as a fixture def cleanup(): if os.getenv("LIGHTNING_MODEL_STORE_TESTING") and os.path.isdir(STORAGE_DIR): shutil.rmtree(STORAGE_DIR) diff --git a/tests/tests_cloud/test_model.py b/tests/tests_cloud/test_model.py index 48764c9df267e..3f7cef60d1942 100644 --- a/tests/tests_cloud/test_model.py +++ b/tests/tests_cloud/test_model.py @@ -1,6 +1,6 @@ import os -from tests_cloud import _USERNAME, STORAGE_DIR +from tests_cloud import _API_KEY, _PROJECT_ID, _USERNAME, STORAGE_DIR from tests_cloud.helpers import cleanup import pytorch_lightning as pl @@ -11,12 +11,7 @@ def test_model(model_name: str = "boring_model", version: str = "latest"): cleanup() - to_lightning_cloud( - model_name, - model=BoringModel(), - api_key=os.getenv("API_KEY", ""), - project_id=os.getenv("PROJECT_ID", ""), - ) + to_lightning_cloud(model_name, model=BoringModel(), api_key=_API_KEY, project_id=_PROJECT_ID) download_from_lightning_cloud(f"{_USERNAME}/{model_name}") assert os.path.isdir(os.path.join(STORAGE_DIR, _USERNAME, model_name, version)) @@ -28,13 +23,7 @@ def test_model(model_name: str = "boring_model", version: str = "latest"): def test_model_without_progress_bar(model_name: str = "boring_model", version: str = "latest"): cleanup() - to_lightning_cloud( - model_name, - model=BoringModel(), - api_key=os.getenv("API_KEY", ""), - project_id=os.getenv("PROJECT_ID", ""), - progress_bar=False, - ) + to_lightning_cloud(model_name, model=BoringModel(), api_key=_API_KEY, project_id=_PROJECT_ID, progress_bar=False) download_from_lightning_cloud(f"{_USERNAME}/{model_name}", progress_bar=False) assert os.path.isdir(os.path.join(STORAGE_DIR, _USERNAME, model_name, version)) @@ -49,13 +38,7 @@ def test_only_weights(model_name: str = "boring_model_only_weights", version: st model = BoringModel() trainer = pl.Trainer(fast_dev_run=True) trainer.fit(model) - to_lightning_cloud( - model_name, - model=model, - weights_only=True, - api_key=os.getenv("API_KEY", ""), - project_id=os.getenv("PROJECT_ID", ""), - ) + to_lightning_cloud(model_name, model=model, weights_only=True, api_key=_API_KEY, project_id=_PROJECT_ID) download_from_lightning_cloud(f"{_USERNAME}/{model_name}") assert os.path.isdir(os.path.join(STORAGE_DIR, _USERNAME, model_name, version)) @@ -72,12 +55,7 @@ def test_checkpoint_path(model_name: str = "boring_model_only_checkpoint_path", trainer = pl.Trainer(fast_dev_run=True) trainer.fit(model) trainer.save_checkpoint("tmp.ckpt") - to_lightning_cloud( - model_name, - checkpoint_path="tmp.ckpt", - api_key=os.getenv("API_KEY", ""), - project_id=os.getenv("PROJECT_ID", ""), - ) + to_lightning_cloud(model_name, checkpoint_path="tmp.ckpt", api_key=_API_KEY, project_id=_PROJECT_ID) download_from_lightning_cloud(f"{_USERNAME}/{model_name}") assert os.path.isdir(os.path.join(STORAGE_DIR, _USERNAME, model_name, version)) diff --git a/tests/tests_cloud/test_requirements.py b/tests/tests_cloud/test_requirements.py index 47ed98959306a..c6f58f61478ad 100644 --- a/tests/tests_cloud/test_requirements.py +++ b/tests/tests_cloud/test_requirements.py @@ -1,6 +1,6 @@ import os -from tests_cloud import _USERNAME, STORAGE_DIR +from tests_cloud import _API_KEY, _PROJECT_ID, _USERNAME, STORAGE_DIR from tests_cloud.helpers import cleanup from lightning.store import download_from_lightning_cloud, to_lightning_cloud @@ -17,8 +17,8 @@ def test_requirements_as_a_file(version: str = "latest", model_name: str = "bori version=version, model=BoringModel(), requirements_file_path=requirements_file_path, - api_key=os.getenv("API_KEY", ""), - project_id=os.getenv("PROJECT_ID", ""), + api_key=_API_KEY, + project_id=_PROJECT_ID, ) download_from_lightning_cloud(f"{_USERNAME}/{model_name}") @@ -38,8 +38,8 @@ def test_requirements_as_a_list(version: str = "1.0.0", model_name: str = "borin version=version, model=BoringModel(), requirements=requirements_list, - api_key=os.getenv("API_KEY", ""), - project_id=os.getenv("PROJECT_ID", ""), + api_key=_API_KEY, + project_id=_PROJECT_ID, ) download_from_lightning_cloud(f"{_USERNAME}/{model_name}", version=version) diff --git a/tests/tests_cloud/test_source_code.py b/tests/tests_cloud/test_source_code.py index db8f864908433..cce218f015075 100644 --- a/tests/tests_cloud/test_source_code.py +++ b/tests/tests_cloud/test_source_code.py @@ -2,7 +2,7 @@ import os import tempfile -from tests_cloud import _USERNAME, STORAGE_DIR +from tests_cloud import _API_KEY, _PROJECT_ID, _USERNAME, STORAGE_DIR from tests_cloud.helpers import cleanup from lightning.store import download_from_lightning_cloud, to_lightning_cloud @@ -12,21 +12,12 @@ def test_source_code_implicit(model_name: str = "model_test_source_code_implicit"): cleanup() - to_lightning_cloud( - model_name, - model=BoringModel(), - api_key=os.getenv("API_KEY", ""), - project_id=os.getenv("PROJECT_ID", ""), - ) + to_lightning_cloud(model_name, model=BoringModel(), api_key=_API_KEY, project_id=_PROJECT_ID) download_from_lightning_cloud(f"{_USERNAME}/{model_name}") assert os.path.isfile( os.path.join( - STORAGE_DIR, - _USERNAME, - model_name, - "latest", - str(os.path.basename(inspect.getsourcefile(BoringModel))), + STORAGE_DIR, _USERNAME, model_name, "latest", str(os.path.basename(inspect.getsourcefile(BoringModel))) ) ) @@ -34,22 +25,12 @@ def test_source_code_implicit(model_name: str = "model_test_source_code_implicit def test_source_code_saving_disabled(model_name: str = "model_test_source_code_dont_save"): cleanup() - to_lightning_cloud( - model_name, - model=BoringModel(), - api_key=os.getenv("API_KEY", ""), - project_id=os.getenv("PROJECT_ID", ""), - save_code=False, - ) + to_lightning_cloud(model_name, model=BoringModel(), api_key=_API_KEY, project_id=_PROJECT_ID, save_code=False) download_from_lightning_cloud(f"{_USERNAME}/{model_name}") assert not os.path.isfile( os.path.join( - STORAGE_DIR, - _USERNAME, - model_name, - "latest", - str(os.path.basename(inspect.getsourcefile(BoringModel))), + STORAGE_DIR, _USERNAME, model_name, "latest", str(os.path.basename(inspect.getsourcefile(BoringModel))) ) ) @@ -59,23 +40,13 @@ def test_source_code_explicit_relative_folder(model_name: str = "model_test_sour dir_upload_path = f"../{os.path.basename(os.getcwd())}/tests/" to_lightning_cloud( - model_name, - model=BoringModel(), - source_code_path=dir_upload_path, - api_key=os.getenv("API_KEY", ""), - project_id=os.getenv("PROJECT_ID", ""), + model_name, model=BoringModel(), source_code_path=dir_upload_path, api_key=_API_KEY, project_id=_PROJECT_ID ) download_from_lightning_cloud(f"{_USERNAME}/{model_name}") assert os.path.isdir( - os.path.join( - STORAGE_DIR, - _USERNAME, - model_name, - "latest", - os.path.basename(os.path.abspath(dir_upload_path)), - ) + os.path.join(STORAGE_DIR, _USERNAME, model_name, "latest", os.path.basename(os.path.abspath(dir_upload_path))) ) @@ -85,23 +56,13 @@ def test_source_code_explicit_absolute_folder(model_name: str = "model_test_sour with tempfile.TemporaryDirectory() as tmpdir: dir_upload_path = os.path.abspath(tmpdir) to_lightning_cloud( - model_name, - model=BoringModel(), - source_code_path=dir_upload_path, - api_key=os.getenv("API_KEY", ""), - project_id=os.getenv("PROJECT_ID", ""), + model_name, model=BoringModel(), source_code_path=dir_upload_path, api_key=_API_KEY, project_id=_PROJECT_ID ) download_from_lightning_cloud(f"{_USERNAME}/{model_name}") assert os.path.isdir( - os.path.join( - STORAGE_DIR, - _USERNAME, - model_name, - "latest", - os.path.basename(os.path.abspath(dir_upload_path)), - ) + os.path.join(STORAGE_DIR, _USERNAME, model_name, "latest", os.path.basename(os.path.abspath(dir_upload_path))) ) @@ -110,11 +71,7 @@ def test_source_code_explicit_file(model_name: str = "model_test_source_code_exp file_name = os.path.abspath("setup.py") to_lightning_cloud( - model_name, - model=BoringModel(), - source_code_path=file_name, - api_key=os.getenv("API_KEY", ""), - project_id=os.getenv("PROJECT_ID", ""), + model_name, model=BoringModel(), source_code_path=file_name, api_key=_API_KEY, project_id=_PROJECT_ID ) download_from_lightning_cloud(f"{_USERNAME}/{model_name}") diff --git a/tests/tests_cloud/test_versioning.py b/tests/tests_cloud/test_versioning.py index 30e784a397417..e56ea1a5ab9d8 100644 --- a/tests/tests_cloud/test_versioning.py +++ b/tests/tests_cloud/test_versioning.py @@ -2,7 +2,7 @@ import platform import pytest -from tests_cloud import _USERNAME, STORAGE_DIR +from tests_cloud import _API_KEY, _PROJECT_ID, _USERNAME, STORAGE_DIR from tests_cloud.helpers import cleanup from lightning.store.cloud_api import download_from_lightning_cloud, to_lightning_cloud @@ -10,12 +10,7 @@ def assert_download_successful(username, model_name, version): - folder_name = os.path.join( - STORAGE_DIR, - username, - model_name, - version, - ) + folder_name = os.path.join(STORAGE_DIR, username, model_name, version) assert os.path.isdir(folder_name), f"Folder name: {folder_name} doesn't exist." assert len(os.listdir(folder_name)) != 0 @@ -37,14 +32,7 @@ def assert_download_successful(username, model_name, version): def test_versioning_valid_case(case, expected_case, model_name: str = "boring_model_versioning"): cleanup() - api_key = os.getenv("API_KEY", "") - to_lightning_cloud( - model_name, - version=case, - model=BoringModel(), - api_key=api_key, - project_id=os.getenv("PROJECT_ID", ""), - ) + to_lightning_cloud(model_name, version=case, model=BoringModel(), api_key=_API_KEY, project_id=_PROJECT_ID) download_from_lightning_cloud(f"{_USERNAME}/{model_name}", version=case) assert_download_successful(_USERNAME, model_name, expected_case) @@ -65,14 +53,7 @@ def test_versioning_invalid_case(case, model_name: str = "boring_model_versionin cleanup() with pytest.raises(AssertionError): - api_key = os.getenv("API_KEY", "") - to_lightning_cloud( - model_name, - version=case, - model=BoringModel(), - api_key=api_key, - project_id=os.getenv("PROJECT_ID", ""), - ) + to_lightning_cloud(model_name, version=case, model=BoringModel(), api_key=_API_KEY, project_id=_PROJECT_ID) error = OSError if case == "*" and platform.system() == "Windows" else AssertionError with pytest.raises(error): From 010edd0cfd7aa05ab6c4474a75d7a6d20da99ad0 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 20 Jan 2023 18:03:04 +0100 Subject: [PATCH 21/42] exceptions --- src/lightning/store/authentication.py | 9 ++--- src/lightning/store/cloud_api.py | 9 ++--- src/lightning/store/save.py | 49 +++++++++++++++------------ src/lightning/store/utils.py | 30 ++++++++-------- 4 files changed, 53 insertions(+), 44 deletions(-) diff --git a/src/lightning/store/authentication.py b/src/lightning/store/authentication.py index e0a102f28590e..f319902058e40 100644 --- a/src/lightning/store/authentication.py +++ b/src/lightning/store/authentication.py @@ -37,10 +37,11 @@ def get_username_from_api_key(api_key: str): url=f"{_LIGHTNING_CLOUD_URL}/v1/auth/user", auth=HTTPBasicAuth("lightning", api_key), ) - assert response.status_code == 200, ( - "API_KEY provided is either invalid or wasn't found in the database." - " Please ensure that you passed the correct API_KEY." - ) + if response.status_code != 200: + raise ConnectionRefusedError( + "API_KEY provided is either invalid or wasn't found in the database." + " Please ensure that you passed the correct API_KEY." + ) return json.loads(response.content)["username"] diff --git a/src/lightning/store/cloud_api.py b/src/lightning/store/cloud_api.py index 6bf6802b90170..6226668d5fc2f 100644 --- a/src/lightning/store/cloud_api.py +++ b/src/lightning/store/cloud_api.py @@ -273,10 +273,11 @@ def download_from_lightning_cloud( os.makedirs(output_dir) response = requests.get(f"{_LIGHTNING_CLOUD_URL}/v1/models?name={username}/{model_name}&version={version}") - assert response.status_code == 200, ( - f"Unable to download the model with name {name} and version {version}." - " Maybe reach out to the model owner or check the arguments again?" - ) + if response.status_code != 200: + raise ConnectionRefusedError( + f"Unable to download the model with name {name} and version {version}." + " Maybe reach out to the model owner or check the arguments again?" + ) download_url_response = json.loads(response.content) download_url = download_url_response["downloadUrl"] diff --git a/src/lightning/store/save.py b/src/lightning/store/save.py index f81cfdf515746..c8c214687cb3d 100644 --- a/src/lightning/store/save.py +++ b/src/lightning/store/save.py @@ -34,7 +34,8 @@ def _check_id(id: str): if id[-1] != "/": id += "/" - assert id.count("/") == 2, "The format for the ID should be: /" + if id.count("/") != 2: + raise ValueError("The format for the ID should be: /") return id @@ -71,7 +72,8 @@ def _save_model(name, model, tmpdir, stored, *args, **kwargs): def _save_model_code(name, model_cls, source_code_path, tmpdir, stored): if source_code_path: source_code_path = os.path.abspath(source_code_path) - assert os.path.exists(source_code_path), f"Given path {source_code_path} does not exist." + if not os.path.exists(source_code_path): + raise FileExistsError(f"Given path {source_code_path} does not exist.") # Copy contents to tmpdir folder if os.path.isdir(source_code_path): @@ -83,12 +85,13 @@ def _save_model_code(name, model_cls, source_code_path, tmpdir, stored): shutil.copytree(source_code_path, f"{tmpdir}/{dir_name}/") stored["code"] = {"type": "folder", "path": f"{dir_name}"} else: - assert os.path.splitext(source_code_path)[-1] == ".py", ( - "Expected a Python file or a directory, to be uploaded for model definition," - f" but found {source_code_path}. If your file is not a Python file, and you still" - " want to save it, please consider saving it in a folder and passing the folder" - " path instead." - ) + if os.path.splitext(source_code_path)[-1] != ".py": + raise FileExistsError( + "Expected a Python file or a directory, to be uploaded for model definition," + f" but found {source_code_path}. If your file is not a Python file, and you still" + " want to save it, please consider saving it in a folder and passing the folder" + " path instead." + ) logging.warning( f"NOTE: File: {source_code_path} is being uploaded to the cloud so that the" @@ -104,12 +107,13 @@ def _save_model_code(name, model_cls, source_code_path, tmpdir, stored): # As those will be executed on import model_class_path = inspect.getsourcefile(model_cls) if model_class_path: - assert os.path.splitext(model_class_path)[-1] == ".py", ( - f"The model definition was found in a non-python file ({model_class_path})," - " which is not currently supported (for safety reasons). If your file is not a" - " Python file, and you still want to save it, please consider saving it in a" - " folder and passing the folder path instead." - ) + if os.path.splitext(model_class_path)[-1] != ".py": + raise FileExistsError( + f"The model definition was found in a non-python file ({model_class_path})," + " which is not currently supported (for safety reasons). If your file is not a" + " Python file, and you still want to save it, please consider saving it in a" + " folder and passing the folder path instead." + ) file_name = os.path.basename(model_class_path) logging.warning( @@ -166,10 +170,10 @@ def _get_url(response_content): auth=HTTPBasicAuth(username, api_key), json=json_field, ) - - assert ( - response.status_code == 200 - ), f"Unable to upload content, did you pass correct credentials? Error: {response.content}" + if response.status_code != 200: + raise ConnectionRefusedError( + f"Unable to upload content, did you pass correct credentials? Error: {response.content}" + ) return _get_url(response.content) @@ -276,10 +280,11 @@ def _common_clean_up(): os.path.join(root, filename), ) - assert os.path.isdir(f"{output_dir}"), ( - "Data downloading to the output" - f" directory: {output_dir} failed. Maybe try again or contact the model owner?" - ) + if not os.path.isdir(f"{output_dir}"): + raise NotADirectoryError( + "Data downloading to the output" + f" directory: {output_dir} failed. Maybe try again or contact the model owner?" + ) except Exception as e: _common_clean_up() raise e diff --git a/src/lightning/store/utils.py b/src/lightning/store/utils.py index ebfeac35a38a4..c4a41f5899cf0 100644 --- a/src/lightning/store/utils.py +++ b/src/lightning/store/utils.py @@ -58,29 +58,31 @@ def split_name(name: str, version: str, l_stage: stage): def get_model_data(name: str, version: str): username, model_name, version = split_name(name, version, stage.LOAD) - assert os.path.exists( - _LIGHTNING_STORAGE_FILE - ), f"ERROR: Could not find {_LIGHTNING_STORAGE_FILE} (to be generated after download_from_lightning_cloud(...))" + if not os.path.exists(_LIGHTNING_STORAGE_FILE): + raise NotADirectoryError( + f"Could not find {_LIGHTNING_STORAGE_FILE} (to be generated after download_from_lightning_cloud(...))" + ) with open(_LIGHTNING_STORAGE_FILE) as storage_file: storage_data = json.load(storage_file) - assert username in storage_data, ( - f"No data found for the given username {username}. Make sure to call" - " `download_from_lightning_cloud` before loading" - ) + if username not in storage_data: + raise KeyError( + f"No data found for the given username {username}. Make sure to call" + " `download_from_lightning_cloud` before loading" + ) user_data = storage_data[username] - assert model_name in user_data, ( - f"No data found for the given model name: {model_name} for the given" - f" username: {username}. Make sure to call `download_from_lightning_cloud` before loading" - ) + if model_name not in user_data: + raise KeyError( + f"No data found for the given model name: {model_name} for the given" + f" username: {username}. Make sure to call `download_from_lightning_cloud` before loading" + ) model_data = user_data[model_name] version = version or "latest" - assert ( - version in model_data - ), f"No data found for the given version: {version}, did you download the model successfully?" + if version not in model_data: + raise KeyError(f"No data found for the given version: {version}, did you download the model successfully?") model_version_data = model_data[version] return model_version_data From 3283ef96c9c3e0a4382e95370c1235f26498b0ce Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 20 Jan 2023 18:26:34 +0100 Subject: [PATCH 22/42] another pass --- .azure/app-cloud-store.yml | 2 +- src/lightning/store/authentication.py | 5 +- src/lightning/store/cloud_api.py | 47 +++++++++--------- src/lightning/store/save.py | 69 +++++++++++++-------------- 4 files changed, 60 insertions(+), 63 deletions(-) diff --git a/.azure/app-cloud-store.yml b/.azure/app-cloud-store.yml index f9a5ab8fd4a5a..9619a793f292d 100644 --- a/.azure/app-cloud-store.yml +++ b/.azure/app-cloud-store.yml @@ -63,6 +63,6 @@ jobs: env: API_KEY: $(LIGHTNING_API_KEY_PROD) API_USERNAME: $(LIGHTNING_USERNAME_PROD) - PROJECT_ID: $(LIGHTNING_PROJECT_ID_STORE) + PROJECT_ID: $(LIGHTNING_PROJECT_ID_PROD) LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD) displayName: 'Run the tests' diff --git a/src/lightning/store/authentication.py b/src/lightning/store/authentication.py index f319902058e40..82c568549b5b3 100644 --- a/src/lightning/store/authentication.py +++ b/src/lightning/store/authentication.py @@ -33,10 +33,7 @@ def _get_user_details(): def get_username_from_api_key(api_key: str): - response = requests.get( - url=f"{_LIGHTNING_CLOUD_URL}/v1/auth/user", - auth=HTTPBasicAuth("lightning", api_key), - ) + response = requests.get(url=f"{_LIGHTNING_CLOUD_URL}/v1/auth/user", auth=HTTPBasicAuth("lightning", api_key)) if response.status_code != 200: raise ConnectionRefusedError( "API_KEY provided is either invalid or wasn't found in the database." diff --git a/src/lightning/store/cloud_api.py b/src/lightning/store/cloud_api.py index 6226668d5fc2f..ef00115460b02 100644 --- a/src/lightning/store/cloud_api.py +++ b/src/lightning/store/cloud_api.py @@ -108,8 +108,10 @@ def to_lightning_cloud( raise ValueError( """" You either need to pass the model or the checkpoint path that you want to save. :) - Any one of: `to_lightning_cloud("model_name", model=modelObj, ...)` - or `to_lightning_cloud("model_name", checkpoint_path="your_checkpoint_path.ckpt", ...)` + Any one of: + `to_lightning_cloud("model_name", model=modelObj, ...)` + or + `to_lightning_cloud("model_name", checkpoint_path="your_checkpoint_path.ckpt", ...)` is required. """ ) @@ -152,8 +154,8 @@ def to_lightning_cloud( if requirements and requirements_file_path: # TODO: Later on, figure out how to merge requirements from both args logging.warning( - "You provided a requirements file (requirements_file_path=...)" - " and requirements list (requirements=...). In case of any collisions," + "You provided a requirements file (..., requirements_file_path=...)" + " and requirements list (..., requirements=...). In case of any collisions," " anything that comes from requirements=... will be given the priority." ) @@ -186,21 +188,20 @@ def to_lightning_cloud( msg = "Finished storing the following data items to the Lightning Cloud.\n" for key, val in stored.items(): if key == "code": - if val["type"] == "file": - msg += f"Stored code as a file with name: {val['path']}\n" - else: - msg += f"Stored code as a folder with name: {val['path']}\n" + msg += f"Stored code as a {val['type']} with name: {val['path']}\n" else: msg += f"Stored {key} with name: {val}\n" - msg += ( - f'\nJust do: download_from_lightning_cloud("{username_from_api_key}/{model_name}", ' - f'version="{version}") in order to download the model from the cloud to your local system.' - ) - msg += ( - f'\nAnd: to_lightning_cloud("{username_from_api_key}/{model_name}", ' - f'version="{version}") in order to load the downloaded model.' - ) + msg += """ + Just do: + `download_from_lightning_cloud("{username_from_api_key}/{model_name}", version="{version}")` + in order to download the model from the cloud to your local system. + """ + msg += """ + And: + `to_lightning_cloud("{username_from_api_key}/{model_name}", version="{version}")` + in order to load the downloaded model. + """ logging.info(msg) @@ -211,8 +212,8 @@ def _load_model(stored, output_dir, *args, **kwargs): return model else: raise ValueError( - "Couldn't find the model when uploaded to our storage. Please check" - " with the model owner to confirm that the models exist in the storage." + "Couldn't find the model when uploaded to our storage." + " Please check with the model owner to confirm that the models exist in the storage." ) @@ -222,8 +223,7 @@ def _load_weights(model, stored, output_dir, *args, **kwargs): return model else: raise ValueError( - "Weights were not found, please contact the model owner to verify if the" - " weights were stored successfully..." + "Weights were not found, please contact the model's owner to verify if the weights were stored correctly." ) @@ -257,6 +257,8 @@ def download_from_lightning_cloud( The target directory, where the model and other data will be stored. If not passed, the data will be stored in `$HOME/.lightning/lightning_model_store///`. (`version` defaults to `latest`) + progress_bar: + Show progress on download. """ version = version or "latest" username, model_name, version = split_name(name, version=version, l_stage=stage.DOWNLOAD) @@ -312,10 +314,9 @@ def download_from_lightning_cloud( logging.info("Downloading done...") logging.info( - f"The source code for your model has been written to {output_dir} folder, and" - f" linked to {linked_output_dir} folder." + f"The source code for your model has been written to {output_dir} folder," + f" and linked to {linked_output_dir} folder." ) - logging.info( "Please make sure to add imports to the necessary classes needed for instantiation of" " your model before calling `load_from_lightning_cloud`." diff --git a/src/lightning/store/save.py b/src/lightning/store/save.py index c8c214687cb3d..32dbce934fe4f 100644 --- a/src/lightning/store/save.py +++ b/src/lightning/store/save.py @@ -231,29 +231,34 @@ def upload_from_file(src, dst): requests.put(url, data=open(archive_path, "rb")) -def _download_and_extract_data_to(output_dir: str, download_url: str, progress_bar: bool): - def _common_clean_up(): - data_file_path = f"{output_dir}/data.tar.gz" - dir_file_path = f"{output_dir}/extracted" - if os.path.exists(data_file_path): - os.remove(data_file_path) - shutil.rmtree(dir_file_path) +def _download_tarfile(download_url: str, output_dir: str, progress_bar: bool) -> None: + with requests.get(download_url, stream=True) as req_stream: + total_size_in_bytes = int(req_stream.headers.get("content-length", 0)) + block_size = 1024 # 1 Kibibyte + + download_progress_bar = None + if progress_bar: + download_progress_bar = tqdm(total=total_size_in_bytes, unit="iB", unit_scale=True) + with open(f"{output_dir}/data.tar.gz", "wb") as f: + for chunk in req_stream.iter_content(chunk_size=block_size): + if download_progress_bar: + download_progress_bar.update(len(chunk)) + f.write(chunk) + if download_progress_bar: + download_progress_bar.close() + + +def _common_clean_up(output_dir: str): + data_file_path = f"{output_dir}/data.tar.gz" + dir_file_path = f"{output_dir}/extracted" + if os.path.exists(data_file_path): + os.remove(data_file_path) + shutil.rmtree(dir_file_path) + +def _download_and_extract_data_to(output_dir: str, download_url: str, progress_bar: bool): try: - with requests.get(download_url, stream=True) as req_stream: - total_size_in_bytes = int(req_stream.headers.get("content-length", 0)) - block_size = 1024 # 1 Kibibyte - - download_progress_bar = None - if progress_bar: - download_progress_bar = tqdm(total=total_size_in_bytes, unit="iB", unit_scale=True) - with open(f"{output_dir}/data.tar.gz", "wb") as f: - for chunk in req_stream.iter_content(chunk_size=block_size): - if download_progress_bar: - download_progress_bar.update(len(chunk)) - f.write(chunk) - if download_progress_bar: - download_progress_bar.close() + _download_tarfile(download_url, output_dir, progress_bar) tar = tarfile.open(f"{output_dir}/data.tar.gz", "r:gz") tmpdir_name = tar.getnames()[0] @@ -263,10 +268,7 @@ def _common_clean_up(): root = f"{output_dir}" for filename in os.listdir(os.path.join(root, "extracted", tmpdir_name)): abs_file_name = os.path.join(root, "extracted", tmpdir_name, filename) - if os.path.isdir(abs_file_name): - func = shutil.copytree - else: - func = shutil.copy + func = shutil.copytree if os.path.isdir(abs_file_name) else shutil.copy dst_file_name = os.path.join(root, filename) if os.path.exists(dst_file_name): @@ -275,21 +277,18 @@ def _common_clean_up(): else: os.remove(dst_file_name) - func( - abs_file_name, - os.path.join(root, filename), - ) + func(abs_file_name, os.path.join(root, filename)) if not os.path.isdir(f"{output_dir}"): raise NotADirectoryError( - "Data downloading to the output" - f" directory: {output_dir} failed. Maybe try again or contact the model owner?" + f"Data downloading to the output directory: {output_dir} failed." + f" Maybe try again or contact the model owner?" ) - except Exception as e: - _common_clean_up() - raise e + except Exception as ex: + _common_clean_up(output_dir) + raise ex else: - _common_clean_up() + _common_clean_up(output_dir) def get_linked_output_dir(src_dir: str): From 8d873d13d91adf41adf19eb08ec03a596a5827ba Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 20 Jan 2023 18:45:27 +0100 Subject: [PATCH 23/42] error --- src/lightning/store/save.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning/store/save.py b/src/lightning/store/save.py index 32dbce934fe4f..70888efcc4f42 100644 --- a/src/lightning/store/save.py +++ b/src/lightning/store/save.py @@ -172,7 +172,7 @@ def _get_url(response_content): ) if response.status_code != 200: raise ConnectionRefusedError( - f"Unable to upload content, did you pass correct credentials? Error: {response.content}" + f"Unable to upload content.\n Error: {response.content}\n for load: {json_field}" ) return _get_url(response.content) From a8308df319baba3f0aaaf09a77237aff9a2e5161 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 20 Jan 2023 17:46:30 +0000 Subject: [PATCH 24/42] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/lightning/store/save.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/lightning/store/save.py b/src/lightning/store/save.py index 70888efcc4f42..621c48f0268eb 100644 --- a/src/lightning/store/save.py +++ b/src/lightning/store/save.py @@ -171,9 +171,7 @@ def _get_url(response_content): json=json_field, ) if response.status_code != 200: - raise ConnectionRefusedError( - f"Unable to upload content.\n Error: {response.content}\n for load: {json_field}" - ) + raise ConnectionRefusedError(f"Unable to upload content.\n Error: {response.content}\n for load: {json_field}") return _get_url(response.content) From 741b33bd1c0a0856d5f95e79f4ff824ed28e04cc Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 20 Jan 2023 19:03:27 +0100 Subject: [PATCH 25/42] assert --- tests/tests_cloud/test_versioning.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tests_cloud/test_versioning.py b/tests/tests_cloud/test_versioning.py index e56ea1a5ab9d8..a61bf384d0e19 100644 --- a/tests/tests_cloud/test_versioning.py +++ b/tests/tests_cloud/test_versioning.py @@ -52,9 +52,9 @@ def test_versioning_valid_case(case, expected_case, model_name: str = "boring_mo def test_versioning_invalid_case(case, model_name: str = "boring_model_versioning"): cleanup() - with pytest.raises(AssertionError): + with pytest.raises(ConnectionRefusedError): to_lightning_cloud(model_name, version=case, model=BoringModel(), api_key=_API_KEY, project_id=_PROJECT_ID) - error = OSError if case == "*" and platform.system() == "Windows" else AssertionError + error = OSError if case == "*" and platform.system() == "Windows" else ConnectionRefusedError with pytest.raises(error): download_from_lightning_cloud(f"{_USERNAME}/{model_name}", version=case) From 84c1591c8ad4b4376b4782178afa13cd55644f1f Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 20 Jan 2023 19:22:29 +0100 Subject: [PATCH 26/42] paths --- tests/tests_cloud/__init__.py | 2 ++ tests/tests_cloud/test_requirements.py | 4 ++-- tests/tests_cloud/test_source_code.py | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/tests_cloud/__init__.py b/tests/tests_cloud/__init__.py index 0ed8c529d9f88..02b30588e7080 100644 --- a/tests/tests_cloud/__init__.py +++ b/tests/tests_cloud/__init__.py @@ -1,5 +1,7 @@ import os +_TEST_ROOT = os.path.dirname(__file__) +_PROJECT_ROOT = os.path.dirname( os.path.dirname(_TEST_ROOT)) _LIGHTNING_DIR = f"{os.path.expanduser('~')}/.lightning" if os.getenv("LIGHTNING_MODEL_STORE_TESTING") == "1": diff --git a/tests/tests_cloud/test_requirements.py b/tests/tests_cloud/test_requirements.py index c6f58f61478ad..dd312aec512ec 100644 --- a/tests/tests_cloud/test_requirements.py +++ b/tests/tests_cloud/test_requirements.py @@ -1,6 +1,6 @@ import os -from tests_cloud import _API_KEY, _PROJECT_ID, _USERNAME, STORAGE_DIR +from tests_cloud import _API_KEY, _PROJECT_ID, _PROJECT_ROOT, _USERNAME, STORAGE_DIR from tests_cloud.helpers import cleanup from lightning.store import download_from_lightning_cloud, to_lightning_cloud @@ -10,7 +10,7 @@ def test_requirements_as_a_file(version: str = "latest", model_name: str = "boring_model"): cleanup() - requirements_file_path = "tests/requirements.txt" + requirements_file_path = os.path.join(_PROJECT_ROOT, "requirements", "app", "base.txt") to_lightning_cloud( model_name, diff --git a/tests/tests_cloud/test_source_code.py b/tests/tests_cloud/test_source_code.py index cce218f015075..4f011930488df 100644 --- a/tests/tests_cloud/test_source_code.py +++ b/tests/tests_cloud/test_source_code.py @@ -2,7 +2,7 @@ import os import tempfile -from tests_cloud import _API_KEY, _PROJECT_ID, _USERNAME, STORAGE_DIR +from tests_cloud import _API_KEY, _PROJECT_ID, _PROJECT_ROOT, _TEST_ROOT, _USERNAME, STORAGE_DIR from tests_cloud.helpers import cleanup from lightning.store import download_from_lightning_cloud, to_lightning_cloud @@ -38,7 +38,7 @@ def test_source_code_saving_disabled(model_name: str = "model_test_source_code_d def test_source_code_explicit_relative_folder(model_name: str = "model_test_source_code_explicit_relative"): cleanup() - dir_upload_path = f"../{os.path.basename(os.getcwd())}/tests/" + dir_upload_path = _TEST_ROOT to_lightning_cloud( model_name, model=BoringModel(), source_code_path=dir_upload_path, api_key=_API_KEY, project_id=_PROJECT_ID ) @@ -69,7 +69,7 @@ def test_source_code_explicit_absolute_folder(model_name: str = "model_test_sour def test_source_code_explicit_file(model_name: str = "model_test_source_code_explicit_file"): cleanup() - file_name = os.path.abspath("setup.py") + file_name = os.path.join(_PROJECT_ROOT, "setup.py") to_lightning_cloud( model_name, model=BoringModel(), source_code_path=file_name, api_key=_API_KEY, project_id=_PROJECT_ID ) From 3d8e35fef87b07d0bea77f61ce7a87762659e2b9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 20 Jan 2023 18:24:43 +0000 Subject: [PATCH 27/42] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/tests_cloud/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_cloud/__init__.py b/tests/tests_cloud/__init__.py index 02b30588e7080..d63255277e318 100644 --- a/tests/tests_cloud/__init__.py +++ b/tests/tests_cloud/__init__.py @@ -1,7 +1,7 @@ import os _TEST_ROOT = os.path.dirname(__file__) -_PROJECT_ROOT = os.path.dirname( os.path.dirname(_TEST_ROOT)) +_PROJECT_ROOT = os.path.dirname(os.path.dirname(_TEST_ROOT)) _LIGHTNING_DIR = f"{os.path.expanduser('~')}/.lightning" if os.getenv("LIGHTNING_MODEL_STORE_TESTING") == "1": From 1f003b8c5d2132106f2223c7c2112d25367297d4 Mon Sep 17 00:00:00 2001 From: Jirka Date: Tue, 24 Jan 2023 17:23:28 +0100 Subject: [PATCH 28/42] debug --- .azure/app-cloud-store.yml | 8 ++++++++ tests/tests_cloud/test_requirements.py | 8 ++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.azure/app-cloud-store.yml b/.azure/app-cloud-store.yml index 9619a793f292d..1aad87d5230a4 100644 --- a/.azure/app-cloud-store.yml +++ b/.azure/app-cloud-store.yml @@ -54,6 +54,10 @@ jobs: inputs: versionSpec: '3.9' + # TODO: this shall be created by the framework if it is missing + - bash: mkdir -p .lightning + displayName: 'create lightning home' + - bash: pip install -e .[test] -f $(TORCH_URL) displayName: 'Install Lightning & dependencies' @@ -66,3 +70,7 @@ jobs: PROJECT_ID: $(LIGHTNING_PROJECT_ID_PROD) LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD) displayName: 'Run the tests' + + # FIXME: remove debugging + - bash: ls -lh .lightning + condition: always() diff --git a/tests/tests_cloud/test_requirements.py b/tests/tests_cloud/test_requirements.py index dd312aec512ec..5649f69fe8af7 100644 --- a/tests/tests_cloud/test_requirements.py +++ b/tests/tests_cloud/test_requirements.py @@ -24,8 +24,8 @@ def test_requirements_as_a_file(version: str = "latest", model_name: str = "bori download_from_lightning_cloud(f"{_USERNAME}/{model_name}") req_folder_path = os.path.join(STORAGE_DIR, _USERNAME, model_name, version) - assert os.path.isdir(req_folder_path) - assert "requirements.txt" in os.listdir(req_folder_path) + assert os.path.isdir(req_folder_path), "missing: %s" % req_folder_path + assert "requirements.txt" in os.listdir(req_folder_path), "among files: %r" % os.listdir(req_folder_path) def test_requirements_as_a_list(version: str = "1.0.0", model_name: str = "boring_model"): @@ -45,8 +45,8 @@ def test_requirements_as_a_list(version: str = "1.0.0", model_name: str = "borin download_from_lightning_cloud(f"{_USERNAME}/{model_name}", version=version) req_folder_path = os.path.join(STORAGE_DIR, _USERNAME, model_name, version) - assert os.path.isdir(req_folder_path) - assert "requirements.txt" in os.listdir(req_folder_path) + assert os.path.isdir(req_folder_path), "missing: %s" % req_folder_path + assert "requirements.txt" in os.listdir(req_folder_path), "among files: %r" % os.listdir(req_folder_path) with open(f"{req_folder_path}/requirements.txt") as req_file: reqs = req_file.readlines() From 67827693d37c90ac424772b6025fb265286784bf Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 25 Jan 2023 01:46:31 +0100 Subject: [PATCH 29/42] if --- src/lightning/store/cloud_api.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/lightning/store/cloud_api.py b/src/lightning/store/cloud_api.py index ef00115460b02..c0c897a17e868 100644 --- a/src/lightning/store/cloud_api.py +++ b/src/lightning/store/cloud_api.py @@ -292,9 +292,8 @@ def download_from_lightning_cloud( logging.info(f"Linking the downloaded folder from {output_dir} to {linked_output_dir} folder.") if os.path.islink(linked_output_dir): os.unlink(linked_output_dir) - if os.path.exists(linked_output_dir): - if os.path.isdir(linked_output_dir): - os.rmdir(linked_output_dir) + if os.path.exists(linked_output_dir) and os.path.isdir(linked_output_dir): + os.rmdir(linked_output_dir) os.symlink(output_dir, linked_output_dir) From a1f25678477119b589ecaf09adb182f8ca311be8 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 25 Jan 2023 08:57:36 +0100 Subject: [PATCH 30/42] move CONSTANTs --- .azure/app-cloud-store.yml | 3 ++- src/lightning/store/__init__.py | 7 +++++++ src/lightning/store/authentication.py | 2 +- src/lightning/store/cloud_api.py | 10 ++-------- src/lightning/store/save.py | 2 +- src/lightning/store/utils.py | 8 ++------ tests/tests_cloud/__init__.py | 2 +- 7 files changed, 16 insertions(+), 18 deletions(-) diff --git a/.azure/app-cloud-store.yml b/.azure/app-cloud-store.yml index 1aad87d5230a4..346c368dd0754 100644 --- a/.azure/app-cloud-store.yml +++ b/.azure/app-cloud-store.yml @@ -47,7 +47,8 @@ jobs: clean: all variables: FREEZE_REQUIREMENTS: "1" - LIGHTNING_MODEL_STORE_TESTING: "1" + # TODO: this seems to be wrongly used in CI + # LIGHTNING_MODEL_STORE_TESTING: "1" TORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html" steps: - task: UsePythonVersion@0 diff --git a/src/lightning/store/__init__.py b/src/lightning/store/__init__.py index 31ce7ade8ef96..0b66490f719a1 100644 --- a/src/lightning/store/__init__.py +++ b/src/lightning/store/__init__.py @@ -1,3 +1,10 @@ +import os + from lightning.store.cloud_api import download_from_lightning_cloud, load_from_lightning_cloud, to_lightning_cloud +_LIGHTNING_DIR = os.path.join(os.path.expanduser("~"), ".lightning") +_LIGHTNING_STORAGE_FILE = os.path.join(_LIGHTNING_DIR, ".lightning_model_storage") +_LIGHTNING_STORAGE_DIR = os.path.join(_LIGHTNING_DIR, "lightning_model_store") +_LIGHTNING_CLOUD_URL = os.getenv("LIGHTNING_CLOUD_URL", default="https://lightning.ai") + __all__ = ["download_from_lightning_cloud", "load_from_lightning_cloud", "to_lightning_cloud"] diff --git a/src/lightning/store/authentication.py b/src/lightning/store/authentication.py index 82c568549b5b3..86a7bbc814ce4 100644 --- a/src/lightning/store/authentication.py +++ b/src/lightning/store/authentication.py @@ -19,7 +19,7 @@ from requests.models import HTTPBasicAuth from lightning.app.utilities.network import LightningClient -from lightning.store.utils import _LIGHTNING_CLOUD_URL +from lightning.store import _LIGHTNING_CLOUD_URL def get_user_details(): diff --git a/src/lightning/store/cloud_api.py b/src/lightning/store/cloud_api.py index c0c897a17e868..099a765087f19 100644 --- a/src/lightning/store/cloud_api.py +++ b/src/lightning/store/cloud_api.py @@ -24,6 +24,7 @@ import lightning as L import pytorch_lightning as PL +from lightning.store import _LIGHTNING_CLOUD_URL, _LIGHTNING_STORAGE_DIR, _LIGHTNING_STORAGE_FILE from lightning.store.authentication import authenticate from lightning.store.save import ( _download_and_extract_data_to, @@ -37,14 +38,7 @@ _write_and_save_requirements, get_linked_output_dir, ) -from lightning.store.utils import ( - _LIGHTNING_CLOUD_URL, - _LIGHTNING_STORAGE_DIR, - _LIGHTNING_STORAGE_FILE, - get_model_data, - split_name, - stage, -) +from lightning.store.utils import get_model_data, split_name, stage logging.basicConfig(level=logging.INFO) diff --git a/src/lightning/store/save.py b/src/lightning/store/save.py index 621c48f0268eb..12695ac0cab0f 100644 --- a/src/lightning/store/save.py +++ b/src/lightning/store/save.py @@ -26,7 +26,7 @@ from tqdm import tqdm from tqdm.utils import CallbackIOWrapper -from lightning.store.utils import _LIGHTNING_CLOUD_URL +from lightning.store import _LIGHTNING_CLOUD_URL logging.basicConfig(level=logging.INFO) diff --git a/src/lightning/store/utils.py b/src/lightning/store/utils.py index c4a41f5899cf0..5d26653339583 100644 --- a/src/lightning/store/utils.py +++ b/src/lightning/store/utils.py @@ -16,6 +16,8 @@ import os from enum import Enum +from lightning.store import _LIGHTNING_STORAGE_FILE + class stage(Enum): UPLOAD = 0 @@ -23,12 +25,6 @@ class stage(Enum): DOWNLOAD = 2 -_LIGHTNING_DIR = os.path.join(os.path.expanduser("~"), ".lightning") -_LIGHTNING_STORAGE_FILE = os.path.join(_LIGHTNING_DIR, ".lightning_model_storage") -_LIGHTNING_STORAGE_DIR = os.path.join(_LIGHTNING_DIR, "lightning_model_store") -_LIGHTNING_CLOUD_URL = os.getenv("LIGHTNING_CLOUD_URL", default="https://lightning.ai") - - def _check_version(version: str): allowed_chars = "0123456789." if version == "latest": diff --git a/tests/tests_cloud/__init__.py b/tests/tests_cloud/__init__.py index d63255277e318..801468fb1b0b3 100644 --- a/tests/tests_cloud/__init__.py +++ b/tests/tests_cloud/__init__.py @@ -7,7 +7,7 @@ if os.getenv("LIGHTNING_MODEL_STORE_TESTING") == "1": STORAGE_DIR = f"{_LIGHTNING_DIR}/lightning_test_model_store/" else: - from lightning.store.utils import _LIGHTNING_STORAGE_DIR as STORAGE_DIR + from lightning.store import _LIGHTNING_STORAGE_DIR as STORAGE_DIR _USERNAME = os.getenv("API_USERNAME", "") assert _USERNAME, "No API_USERNAME env variable, make sure to add it before testing" From 94c90adc923f9a4a206c3f419fc5976cd434e55e Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 25 Jan 2023 13:47:24 +0100 Subject: [PATCH 31/42] imports --- src/lightning/store/__init__.py | 7 ------- src/lightning/store/authentication.py | 4 +++- src/lightning/store/cloud_api.py | 5 +++-- src/lightning/store/save.py | 6 +++++- src/lightning/store/utils.py | 2 +- tests/tests_cloud/__init__.py | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/lightning/store/__init__.py b/src/lightning/store/__init__.py index 0b66490f719a1..31ce7ade8ef96 100644 --- a/src/lightning/store/__init__.py +++ b/src/lightning/store/__init__.py @@ -1,10 +1,3 @@ -import os - from lightning.store.cloud_api import download_from_lightning_cloud, load_from_lightning_cloud, to_lightning_cloud -_LIGHTNING_DIR = os.path.join(os.path.expanduser("~"), ".lightning") -_LIGHTNING_STORAGE_FILE = os.path.join(_LIGHTNING_DIR, ".lightning_model_storage") -_LIGHTNING_STORAGE_DIR = os.path.join(_LIGHTNING_DIR, "lightning_model_store") -_LIGHTNING_CLOUD_URL = os.getenv("LIGHTNING_CLOUD_URL", default="https://lightning.ai") - __all__ = ["download_from_lightning_cloud", "load_from_lightning_cloud", "to_lightning_cloud"] diff --git a/src/lightning/store/authentication.py b/src/lightning/store/authentication.py index 86a7bbc814ce4..cd010f66b13bc 100644 --- a/src/lightning/store/authentication.py +++ b/src/lightning/store/authentication.py @@ -13,13 +13,15 @@ # limitations under the License. import json +import os import webbrowser import requests from requests.models import HTTPBasicAuth from lightning.app.utilities.network import LightningClient -from lightning.store import _LIGHTNING_CLOUD_URL + +_LIGHTNING_CLOUD_URL = os.getenv("LIGHTNING_CLOUD_URL", default="https://lightning.ai") def get_user_details(): diff --git a/src/lightning/store/cloud_api.py b/src/lightning/store/cloud_api.py index 099a765087f19..59e911a0ceb4e 100644 --- a/src/lightning/store/cloud_api.py +++ b/src/lightning/store/cloud_api.py @@ -24,10 +24,11 @@ import lightning as L import pytorch_lightning as PL -from lightning.store import _LIGHTNING_CLOUD_URL, _LIGHTNING_STORAGE_DIR, _LIGHTNING_STORAGE_FILE -from lightning.store.authentication import authenticate +from lightning.store.authentication import _LIGHTNING_CLOUD_URL, authenticate from lightning.store.save import ( _download_and_extract_data_to, + _LIGHTNING_STORAGE_DIR, + _LIGHTNING_STORAGE_FILE, _save_checkpoint_from_path, _save_meta_data, _save_model, diff --git a/src/lightning/store/save.py b/src/lightning/store/save.py index 12695ac0cab0f..00528644fc585 100644 --- a/src/lightning/store/save.py +++ b/src/lightning/store/save.py @@ -26,10 +26,14 @@ from tqdm import tqdm from tqdm.utils import CallbackIOWrapper -from lightning.store import _LIGHTNING_CLOUD_URL +from lightning.store.authentication import _LIGHTNING_CLOUD_URL logging.basicConfig(level=logging.INFO) +_LIGHTNING_DIR = os.path.join(os.path.expanduser("~"), ".lightning") +_LIGHTNING_STORAGE_FILE = os.path.join(_LIGHTNING_DIR, ".lightning_model_storage") +_LIGHTNING_STORAGE_DIR = os.path.join(_LIGHTNING_DIR, "lightning_model_store") + def _check_id(id: str): if id[-1] != "/": diff --git a/src/lightning/store/utils.py b/src/lightning/store/utils.py index 5d26653339583..3710b8877c5f7 100644 --- a/src/lightning/store/utils.py +++ b/src/lightning/store/utils.py @@ -16,7 +16,7 @@ import os from enum import Enum -from lightning.store import _LIGHTNING_STORAGE_FILE +from lightning.store.save import _LIGHTNING_STORAGE_FILE class stage(Enum): diff --git a/tests/tests_cloud/__init__.py b/tests/tests_cloud/__init__.py index 801468fb1b0b3..08af49aff3938 100644 --- a/tests/tests_cloud/__init__.py +++ b/tests/tests_cloud/__init__.py @@ -7,7 +7,7 @@ if os.getenv("LIGHTNING_MODEL_STORE_TESTING") == "1": STORAGE_DIR = f"{_LIGHTNING_DIR}/lightning_test_model_store/" else: - from lightning.store import _LIGHTNING_STORAGE_DIR as STORAGE_DIR + from lightning.store.save import _LIGHTNING_STORAGE_DIR as STORAGE_DIR _USERNAME = os.getenv("API_USERNAME", "") assert _USERNAME, "No API_USERNAME env variable, make sure to add it before testing" From acc2b6e97d29a13269c7bb97ba87c9a6973c05e9 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 25 Jan 2023 14:01:44 +0100 Subject: [PATCH 32/42] drop test dev --- .azure/app-cloud-store.yml | 2 -- tests/tests_cloud/__init__.py | 6 ------ tests/tests_cloud/helpers.py | 8 +++++--- tests/tests_cloud/test_model.py | 11 ++++++----- tests/tests_cloud/test_requirements.py | 7 ++++--- tests/tests_cloud/test_source_code.py | 25 +++++++++++++++++++------ tests/tests_cloud/test_versioning.py | 5 +++-- 7 files changed, 37 insertions(+), 27 deletions(-) diff --git a/.azure/app-cloud-store.yml b/.azure/app-cloud-store.yml index 346c368dd0754..fda24f3c97f61 100644 --- a/.azure/app-cloud-store.yml +++ b/.azure/app-cloud-store.yml @@ -47,8 +47,6 @@ jobs: clean: all variables: FREEZE_REQUIREMENTS: "1" - # TODO: this seems to be wrongly used in CI - # LIGHTNING_MODEL_STORE_TESTING: "1" TORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html" steps: - task: UsePythonVersion@0 diff --git a/tests/tests_cloud/__init__.py b/tests/tests_cloud/__init__.py index 08af49aff3938..70245585b8152 100644 --- a/tests/tests_cloud/__init__.py +++ b/tests/tests_cloud/__init__.py @@ -4,10 +4,6 @@ _PROJECT_ROOT = os.path.dirname(os.path.dirname(_TEST_ROOT)) _LIGHTNING_DIR = f"{os.path.expanduser('~')}/.lightning" -if os.getenv("LIGHTNING_MODEL_STORE_TESTING") == "1": - STORAGE_DIR = f"{_LIGHTNING_DIR}/lightning_test_model_store/" -else: - from lightning.store.save import _LIGHTNING_STORAGE_DIR as STORAGE_DIR _USERNAME = os.getenv("API_USERNAME", "") assert _USERNAME, "No API_USERNAME env variable, make sure to add it before testing" @@ -15,5 +11,3 @@ assert _API_KEY, "No API_KEY env variable, make sure to add it before testing" _PROJECT_ID = os.getenv("PROJECT_ID", "") assert _PROJECT_ID, "No PROJECT_ID env variable, make sure to add it before testing" - -__all__ = ["STORAGE_DIR"] diff --git a/tests/tests_cloud/helpers.py b/tests/tests_cloud/helpers.py index 7af0dda1bd31b..282cb095dfb77 100644 --- a/tests/tests_cloud/helpers.py +++ b/tests/tests_cloud/helpers.py @@ -1,10 +1,12 @@ import os import shutil -from tests_cloud import STORAGE_DIR +from lightning.store.save import _LIGHTNING_STORAGE_DIR # TODO: make this as a fixture def cleanup(): - if os.getenv("LIGHTNING_MODEL_STORE_TESTING") and os.path.isdir(STORAGE_DIR): - shutil.rmtree(STORAGE_DIR) + # todo: `LIGHTNING_MODEL_STORE_TESTING` is nor working as intended, + # so the fixture shall create temp folder and map it home... + if os.getenv("LIGHTNING_MODEL_STORE_TESTING") and os.path.isdir(_LIGHTNING_STORAGE_DIR): + shutil.rmtree(_LIGHTNING_STORAGE_DIR) diff --git a/tests/tests_cloud/test_model.py b/tests/tests_cloud/test_model.py index 3f7cef60d1942..c9ad3b9e5ecc3 100644 --- a/tests/tests_cloud/test_model.py +++ b/tests/tests_cloud/test_model.py @@ -1,10 +1,11 @@ import os -from tests_cloud import _API_KEY, _PROJECT_ID, _USERNAME, STORAGE_DIR +from tests_cloud import _API_KEY, _PROJECT_ID, _USERNAME from tests_cloud.helpers import cleanup import pytorch_lightning as pl from lightning.store import download_from_lightning_cloud, load_from_lightning_cloud, to_lightning_cloud +from lightning.store.save import _LIGHTNING_STORAGE_DIR from pytorch_lightning.demos.boring_classes import BoringModel @@ -14,7 +15,7 @@ def test_model(model_name: str = "boring_model", version: str = "latest"): to_lightning_cloud(model_name, model=BoringModel(), api_key=_API_KEY, project_id=_PROJECT_ID) download_from_lightning_cloud(f"{_USERNAME}/{model_name}") - assert os.path.isdir(os.path.join(STORAGE_DIR, _USERNAME, model_name, version)) + assert os.path.isdir(os.path.join(_LIGHTNING_STORAGE_DIR, _USERNAME, model_name, version)) model = load_from_lightning_cloud(f"{_USERNAME}/{model_name}") assert model is not None @@ -26,7 +27,7 @@ def test_model_without_progress_bar(model_name: str = "boring_model", version: s to_lightning_cloud(model_name, model=BoringModel(), api_key=_API_KEY, project_id=_PROJECT_ID, progress_bar=False) download_from_lightning_cloud(f"{_USERNAME}/{model_name}", progress_bar=False) - assert os.path.isdir(os.path.join(STORAGE_DIR, _USERNAME, model_name, version)) + assert os.path.isdir(os.path.join(_LIGHTNING_STORAGE_DIR, _USERNAME, model_name, version)) model = load_from_lightning_cloud(f"{_USERNAME}/{model_name}") assert model is not None @@ -41,7 +42,7 @@ def test_only_weights(model_name: str = "boring_model_only_weights", version: st to_lightning_cloud(model_name, model=model, weights_only=True, api_key=_API_KEY, project_id=_PROJECT_ID) download_from_lightning_cloud(f"{_USERNAME}/{model_name}") - assert os.path.isdir(os.path.join(STORAGE_DIR, _USERNAME, model_name, version)) + assert os.path.isdir(os.path.join(_LIGHTNING_STORAGE_DIR, _USERNAME, model_name, version)) model_with_weights = load_from_lightning_cloud(f"{_USERNAME}/{model_name}", load_weights=True, model=model) assert model_with_weights is not None @@ -58,7 +59,7 @@ def test_checkpoint_path(model_name: str = "boring_model_only_checkpoint_path", to_lightning_cloud(model_name, checkpoint_path="tmp.ckpt", api_key=_API_KEY, project_id=_PROJECT_ID) download_from_lightning_cloud(f"{_USERNAME}/{model_name}") - assert os.path.isdir(os.path.join(STORAGE_DIR, _USERNAME, model_name, version)) + assert os.path.isdir(os.path.join(_LIGHTNING_STORAGE_DIR, _USERNAME, model_name, version)) ckpt = load_from_lightning_cloud(f"{_USERNAME}/{model_name}", load_checkpoint=True, model=model) assert ckpt is not None diff --git a/tests/tests_cloud/test_requirements.py b/tests/tests_cloud/test_requirements.py index 5649f69fe8af7..92f162ebb8047 100644 --- a/tests/tests_cloud/test_requirements.py +++ b/tests/tests_cloud/test_requirements.py @@ -1,9 +1,10 @@ import os -from tests_cloud import _API_KEY, _PROJECT_ID, _PROJECT_ROOT, _USERNAME, STORAGE_DIR +from tests_cloud import _API_KEY, _PROJECT_ID, _PROJECT_ROOT, _USERNAME from tests_cloud.helpers import cleanup from lightning.store import download_from_lightning_cloud, to_lightning_cloud +from lightning.store.save import _LIGHTNING_STORAGE_DIR from pytorch_lightning.demos.boring_classes import BoringModel @@ -23,7 +24,7 @@ def test_requirements_as_a_file(version: str = "latest", model_name: str = "bori download_from_lightning_cloud(f"{_USERNAME}/{model_name}") - req_folder_path = os.path.join(STORAGE_DIR, _USERNAME, model_name, version) + req_folder_path = os.path.join(_LIGHTNING_STORAGE_DIR, _USERNAME, model_name, version) assert os.path.isdir(req_folder_path), "missing: %s" % req_folder_path assert "requirements.txt" in os.listdir(req_folder_path), "among files: %r" % os.listdir(req_folder_path) @@ -44,7 +45,7 @@ def test_requirements_as_a_list(version: str = "1.0.0", model_name: str = "borin download_from_lightning_cloud(f"{_USERNAME}/{model_name}", version=version) - req_folder_path = os.path.join(STORAGE_DIR, _USERNAME, model_name, version) + req_folder_path = os.path.join(_LIGHTNING_STORAGE_DIR, _USERNAME, model_name, version) assert os.path.isdir(req_folder_path), "missing: %s" % req_folder_path assert "requirements.txt" in os.listdir(req_folder_path), "among files: %r" % os.listdir(req_folder_path) diff --git a/tests/tests_cloud/test_source_code.py b/tests/tests_cloud/test_source_code.py index 4f011930488df..f7dec36389064 100644 --- a/tests/tests_cloud/test_source_code.py +++ b/tests/tests_cloud/test_source_code.py @@ -2,10 +2,11 @@ import os import tempfile -from tests_cloud import _API_KEY, _PROJECT_ID, _PROJECT_ROOT, _TEST_ROOT, _USERNAME, STORAGE_DIR +from tests_cloud import _API_KEY, _PROJECT_ID, _PROJECT_ROOT, _TEST_ROOT, _USERNAME from tests_cloud.helpers import cleanup from lightning.store import download_from_lightning_cloud, to_lightning_cloud +from lightning.store.save import _LIGHTNING_STORAGE_DIR from pytorch_lightning.demos.boring_classes import BoringModel @@ -17,7 +18,11 @@ def test_source_code_implicit(model_name: str = "model_test_source_code_implicit download_from_lightning_cloud(f"{_USERNAME}/{model_name}") assert os.path.isfile( os.path.join( - STORAGE_DIR, _USERNAME, model_name, "latest", str(os.path.basename(inspect.getsourcefile(BoringModel))) + _LIGHTNING_STORAGE_DIR, + _USERNAME, + model_name, + "latest", + str(os.path.basename(inspect.getsourcefile(BoringModel))), ) ) @@ -30,7 +35,11 @@ def test_source_code_saving_disabled(model_name: str = "model_test_source_code_d download_from_lightning_cloud(f"{_USERNAME}/{model_name}") assert not os.path.isfile( os.path.join( - STORAGE_DIR, _USERNAME, model_name, "latest", str(os.path.basename(inspect.getsourcefile(BoringModel))) + _LIGHTNING_STORAGE_DIR, + _USERNAME, + model_name, + "latest", + str(os.path.basename(inspect.getsourcefile(BoringModel))), ) ) @@ -46,7 +55,9 @@ def test_source_code_explicit_relative_folder(model_name: str = "model_test_sour download_from_lightning_cloud(f"{_USERNAME}/{model_name}") assert os.path.isdir( - os.path.join(STORAGE_DIR, _USERNAME, model_name, "latest", os.path.basename(os.path.abspath(dir_upload_path))) + os.path.join( + _LIGHTNING_STORAGE_DIR, _USERNAME, model_name, "latest", os.path.basename(os.path.abspath(dir_upload_path)) + ) ) @@ -62,7 +73,9 @@ def test_source_code_explicit_absolute_folder(model_name: str = "model_test_sour download_from_lightning_cloud(f"{_USERNAME}/{model_name}") assert os.path.isdir( - os.path.join(STORAGE_DIR, _USERNAME, model_name, "latest", os.path.basename(os.path.abspath(dir_upload_path))) + os.path.join( + _LIGHTNING_STORAGE_DIR, _USERNAME, model_name, "latest", os.path.basename(os.path.abspath(dir_upload_path)) + ) ) @@ -78,7 +91,7 @@ def test_source_code_explicit_file(model_name: str = "model_test_source_code_exp assert os.path.isfile( os.path.join( - STORAGE_DIR, + _LIGHTNING_STORAGE_DIR, _USERNAME, model_name, "latest", diff --git a/tests/tests_cloud/test_versioning.py b/tests/tests_cloud/test_versioning.py index a61bf384d0e19..8c52ec3159fcb 100644 --- a/tests/tests_cloud/test_versioning.py +++ b/tests/tests_cloud/test_versioning.py @@ -2,15 +2,16 @@ import platform import pytest -from tests_cloud import _API_KEY, _PROJECT_ID, _USERNAME, STORAGE_DIR +from tests_cloud import _API_KEY, _PROJECT_ID, _USERNAME from tests_cloud.helpers import cleanup from lightning.store.cloud_api import download_from_lightning_cloud, to_lightning_cloud +from lightning.store.save import _LIGHTNING_STORAGE_DIR from pytorch_lightning.demos.boring_classes import BoringModel def assert_download_successful(username, model_name, version): - folder_name = os.path.join(STORAGE_DIR, username, model_name, version) + folder_name = os.path.join(_LIGHTNING_STORAGE_DIR, username, model_name, version) assert os.path.isdir(folder_name), f"Folder name: {folder_name} doesn't exist." assert len(os.listdir(folder_name)) != 0 From 274abdc765249a5b0c4c682e2ae25af3ec68807c Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 25 Jan 2023 14:02:18 +0100 Subject: [PATCH 33/42] ci --- .azure/app-cloud-store.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.azure/app-cloud-store.yml b/.azure/app-cloud-store.yml index fda24f3c97f61..b0dc78ed2e695 100644 --- a/.azure/app-cloud-store.yml +++ b/.azure/app-cloud-store.yml @@ -53,10 +53,6 @@ jobs: inputs: versionSpec: '3.9' - # TODO: this shall be created by the framework if it is missing - - bash: mkdir -p .lightning - displayName: 'create lightning home' - - bash: pip install -e .[test] -f $(TORCH_URL) displayName: 'Install Lightning & dependencies' @@ -69,7 +65,3 @@ jobs: PROJECT_ID: $(LIGHTNING_PROJECT_ID_PROD) LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD) displayName: 'Run the tests' - - # FIXME: remove debugging - - bash: ls -lh .lightning - condition: always() From eff305d9d61d1043fc6bd1219407264f4762668d Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 25 Jan 2023 14:31:27 +0100 Subject: [PATCH 34/42] reuse constants --- src/lightning/store/authentication.py | 4 ++-- src/lightning/store/cloud_api.py | 5 +++-- src/lightning/store/save.py | 4 ++-- src/lightning_app/core/constants.py | 1 + 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/lightning/store/authentication.py b/src/lightning/store/authentication.py index cd010f66b13bc..74ca0b8261bca 100644 --- a/src/lightning/store/authentication.py +++ b/src/lightning/store/authentication.py @@ -13,15 +13,15 @@ # limitations under the License. import json -import os import webbrowser import requests from requests.models import HTTPBasicAuth +from lightning.app.core.constants import get_lightning_cloud_url from lightning.app.utilities.network import LightningClient -_LIGHTNING_CLOUD_URL = os.getenv("LIGHTNING_CLOUD_URL", default="https://lightning.ai") +_LIGHTNING_CLOUD_URL = get_lightning_cloud_url() def get_user_details(): diff --git a/src/lightning/store/cloud_api.py b/src/lightning/store/cloud_api.py index 59e911a0ceb4e..670aec549e777 100644 --- a/src/lightning/store/cloud_api.py +++ b/src/lightning/store/cloud_api.py @@ -24,7 +24,8 @@ import lightning as L import pytorch_lightning as PL -from lightning.store.authentication import _LIGHTNING_CLOUD_URL, authenticate +from lightning.app.core.constants import LIGHTNING_MODELS_PUBLIC_REGISTRY +from lightning.store.authentication import authenticate from lightning.store.save import ( _download_and_extract_data_to, _LIGHTNING_STORAGE_DIR, @@ -269,7 +270,7 @@ def download_from_lightning_cloud( if not os.path.isdir(output_dir): os.makedirs(output_dir) - response = requests.get(f"{_LIGHTNING_CLOUD_URL}/v1/models?name={username}/{model_name}&version={version}") + response = requests.get(f"{LIGHTNING_MODELS_PUBLIC_REGISTRY}?name={username}/{model_name}&version={version}") if response.status_code != 200: raise ConnectionRefusedError( f"Unable to download the model with name {name} and version {version}." diff --git a/src/lightning/store/save.py b/src/lightning/store/save.py index 00528644fc585..44562ab215cd4 100644 --- a/src/lightning/store/save.py +++ b/src/lightning/store/save.py @@ -26,7 +26,7 @@ from tqdm import tqdm from tqdm.utils import CallbackIOWrapper -from lightning.store.authentication import _LIGHTNING_CLOUD_URL +from lightning.app.core.constants import LIGHTNING_MODELS_PUBLIC_REGISTRY logging.basicConfig(level=logging.INFO) @@ -170,7 +170,7 @@ def _get_url(response_content): if project_id: json_field["project_id"] = project_id response = requests.post( - f"{_LIGHTNING_CLOUD_URL}/v1/models", + LIGHTNING_MODELS_PUBLIC_REGISTRY, auth=HTTPBasicAuth(username, api_key), json=json_field, ) diff --git a/src/lightning_app/core/constants.py b/src/lightning_app/core/constants.py index 307f43072b512..76062ea60d200 100644 --- a/src/lightning_app/core/constants.py +++ b/src/lightning_app/core/constants.py @@ -50,6 +50,7 @@ def get_lightning_cloud_url() -> str: DOT_IGNORE_FILENAME = ".lightningignore" LIGHTNING_COMPONENT_PUBLIC_REGISTRY = "https://lightning.ai/v1/components" LIGHTNING_APPS_PUBLIC_REGISTRY = "https://lightning.ai/v1/apps" +LIGHTNING_MODELS_PUBLIC_REGISTRY = "https://lightning.ai/v1/models" # EXPERIMENTAL: ENV VARIABLES TO ENABLE MULTIPLE WORKS IN THE SAME MACHINE DEFAULT_NUMBER_OF_EXPOSED_PORTS = int(os.getenv("DEFAULT_NUMBER_OF_EXPOSED_PORTS", "50")) From 3fa82b0e23490de03850e7b63426dd2cdb284086 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 25 Jan 2023 14:58:32 +0100 Subject: [PATCH 35/42] cleaning --- src/lightning/store/authentication.py | 19 +++++++------------ src/lightning/store/cloud_api.py | 11 +++++------ src/lightning/store/save.py | 25 +++++++++++-------------- src/lightning/store/utils.py | 4 ++-- 4 files changed, 25 insertions(+), 34 deletions(-) diff --git a/src/lightning/store/authentication.py b/src/lightning/store/authentication.py index 74ca0b8261bca..6a7df5faac016 100644 --- a/src/lightning/store/authentication.py +++ b/src/lightning/store/authentication.py @@ -25,13 +25,9 @@ def get_user_details(): - def _get_user_details(): - client = LightningClient() - user_details = client.auth_service_get_user() - return (user_details.username, user_details.api_key) - - username, api_key = _get_user_details() - return username, api_key + client = LightningClient() + user_details = client.auth_service_get_user() + return user_details.username, user_details.api_key def get_username_from_api_key(api_key: str): @@ -47,9 +43,9 @@ def get_username_from_api_key(api_key: str): def _check_browser_runnable(): try: webbrowser.get() - return True except webbrowser.Error: return False + return True def authenticate(inp_api_key: str = ""): @@ -60,8 +56,7 @@ def authenticate(inp_api_key: str = ""): " In order to run the commands on this system, we suggest passing the `api_key`" " after logging into https://lightning.ai." ) - username, api_key = get_user_details() - return username, api_key - - username = get_username_from_api_key(inp_api_key) + username, inp_api_key = get_user_details() + else: + username = get_username_from_api_key(inp_api_key) return username, inp_api_key diff --git a/src/lightning/store/cloud_api.py b/src/lightning/store/cloud_api.py index 670aec549e777..8641988682270 100644 --- a/src/lightning/store/cloud_api.py +++ b/src/lightning/store/cloud_api.py @@ -17,7 +17,7 @@ import os import sys import tempfile -from typing import List, Optional +from typing import Any, List, Optional import requests import torch @@ -58,13 +58,12 @@ def to_lightning_cloud( project_id: str = "", progress_bar: bool = True, save_code: bool = True, - *args, - **kwargs, + *args: Any, + **kwargs: Any, ): """Store model to lightning cloud. Args: - name: The model name. Model/Checkpoint will be uploaded with this unique name. Format: "model_name" version: @@ -318,8 +317,8 @@ def download_from_lightning_cloud( ) -def _validate_output_dir(dir: str): - if not os.path.exists(dir): +def _validate_output_dir(folder: str) -> None: + if not os.path.exists(folder): raise ValueError( "The output directory doesn't exist... did you forget to call download_from_lightning_cloud(...)?" ) diff --git a/src/lightning/store/save.py b/src/lightning/store/save.py index 44562ab215cd4..88c9e31345fdb 100644 --- a/src/lightning/store/save.py +++ b/src/lightning/store/save.py @@ -35,7 +35,7 @@ _LIGHTNING_STORAGE_DIR = os.path.join(_LIGHTNING_DIR, "lightning_model_store") -def _check_id(id: str): +def _check_id(id: str) -> str: if id[-1] != "/": id += "/" if id.count("/") != 2: @@ -43,21 +43,21 @@ def _check_id(id: str): return id -def _save_checkpoint(name, checkpoint, tmpdir, stored): +def _save_checkpoint(name, checkpoint, tmpdir, stored: dict) -> dict: checkpoint_file_path = f"{tmpdir}/checkpoint.ckpt" torch.save(checkpoint, checkpoint_file_path) stored["checkpoint"] = "checkpoint.ckpt" return stored -def _save_checkpoint_from_path(name, path, tmpdir, stored): +def _save_checkpoint_from_path(name, path, tmpdir, stored: dict) -> dict: checkpoint_file_path = path shutil.copy(checkpoint_file_path, f"{tmpdir}/checkpoint.ckpt") stored["checkpoint"] = "checkpoint.ckpt" return stored -def _save_model_weights(name, model_state_dict, tmpdir, stored, *args, **kwargs): +def _save_model_weights(name, model_state_dict, tmpdir, stored, *args, **kwargs) -> dict: # For now we assume that it's always going to be public weights_file_path = f"{tmpdir}/weights.pt" torch.save(model_state_dict, weights_file_path, *args, **kwargs) @@ -65,7 +65,7 @@ def _save_model_weights(name, model_state_dict, tmpdir, stored, *args, **kwargs) return stored -def _save_model(name, model, tmpdir, stored, *args, **kwargs): +def _save_model(name, model, tmpdir, stored, *args, **kwargs) -> dict: # For now we assume that it's always going to be public model_file_path = f"{tmpdir}/model" torch.save(model, model_file_path, *args, **kwargs) @@ -73,7 +73,7 @@ def _save_model(name, model, tmpdir, stored, *args, **kwargs): return stored -def _save_model_code(name, model_cls, source_code_path, tmpdir, stored): +def _save_model_code(name, model_cls, source_code_path, tmpdir, stored) -> dict: if source_code_path: source_code_path = os.path.abspath(source_code_path) if not os.path.exists(source_code_path): @@ -144,7 +144,7 @@ def _write_and_save_requirements(name, requirements, stored, tmpdir): return stored -def _save_requirements_file(name, requirements_file_path, stored, tmpdir): +def _save_requirements_file(name, requirements_file_path, stored, tmpdir) -> dict: shutil.copyfile(os.path.abspath(requirements_file_path), f"{tmpdir}/requirements.txt") stored["requirements"] = requirements_file_path return stored @@ -190,10 +190,7 @@ def _process_stored(stored: dict): processed_dict[f"stored_{key}"] = val return processed_dict - meta_data = { - "cls": model.__class__.__name__, - } - + meta_data = {"cls": model.__class__.__name__} meta_data.update(_process_stored(stored)) return _upload_metadata( @@ -206,7 +203,7 @@ def _process_stored(stored: dict): ) -def _submit_data_to_url(url: str, tmpdir: str, progress_bar: bool): +def _submit_data_to_url(url: str, tmpdir: str, progress_bar: bool) -> None: def _make_tar(tmpdir, archive_output_path): with tarfile.open(archive_output_path, "w:gz") as tar: tar.add(tmpdir) @@ -250,7 +247,7 @@ def _download_tarfile(download_url: str, output_dir: str, progress_bar: bool) -> download_progress_bar.close() -def _common_clean_up(output_dir: str): +def _common_clean_up(output_dir: str) -> None: data_file_path = f"{output_dir}/data.tar.gz" dir_file_path = f"{output_dir}/extracted" if os.path.exists(data_file_path): @@ -258,7 +255,7 @@ def _common_clean_up(output_dir: str): shutil.rmtree(dir_file_path) -def _download_and_extract_data_to(output_dir: str, download_url: str, progress_bar: bool): +def _download_and_extract_data_to(output_dir: str, download_url: str, progress_bar: bool) -> None: try: _download_tarfile(download_url, output_dir, progress_bar) diff --git a/src/lightning/store/utils.py b/src/lightning/store/utils.py index 3710b8877c5f7..f6e609aab6b98 100644 --- a/src/lightning/store/utils.py +++ b/src/lightning/store/utils.py @@ -25,7 +25,7 @@ class stage(Enum): DOWNLOAD = 2 -def _check_version(version: str): +def _check_version(version: str) -> bool: allowed_chars = "0123456789." if version == "latest": return True @@ -48,7 +48,7 @@ def _split_name(name: str, version: str, l_stage: stage): def split_name(name: str, version: str, l_stage: stage): username, model_name, version = _split_name(name, version, l_stage) - return (username, model_name, version) + return username, model_name, version def get_model_data(name: str, version: str): From ef366adf52ef8a751774b4e521eabc95de4c98c8 Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 26 Jan 2023 09:20:34 +0100 Subject: [PATCH 36/42] explicit --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index a34923c07037c..7f4625487f8d9 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,9 @@ wheels/ .installed.cfg *.egg src/*/version.info +src/lightning/app/ +src/lightning/fabric/ +src/lightning/pytorch/ # PyInstaller # Usually these files are written by a python script from a template From 003b25c640877958a57afd0f39bc96488b846d0e Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 26 Jan 2023 10:10:52 +0100 Subject: [PATCH 37/42] todo --- src/lightning/store/authentication.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lightning/store/authentication.py b/src/lightning/store/authentication.py index 6a7df5faac016..0f2d9fd9f4c2d 100644 --- a/src/lightning/store/authentication.py +++ b/src/lightning/store/authentication.py @@ -49,6 +49,8 @@ def _check_browser_runnable(): def authenticate(inp_api_key: str = ""): + # TODO: we have headless login now, + # so it could be reasonable to just point to that if browser can't be opened / user can't be authed if not inp_api_key: if not _check_browser_runnable(): raise ValueError( From 318036f9fea69deb3aa3b133033a19ce5837661d Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Thu, 26 Jan 2023 14:59:25 +0100 Subject: [PATCH 38/42] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrian Wälchli --- src/lightning/store/authentication.py | 6 +++--- src/lightning/store/save.py | 2 +- src/lightning/store/utils.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/lightning/store/authentication.py b/src/lightning/store/authentication.py index 0f2d9fd9f4c2d..49701e04aa214 100644 --- a/src/lightning/store/authentication.py +++ b/src/lightning/store/authentication.py @@ -24,13 +24,13 @@ _LIGHTNING_CLOUD_URL = get_lightning_cloud_url() -def get_user_details(): +def _get_user_details(): client = LightningClient() user_details = client.auth_service_get_user() return user_details.username, user_details.api_key -def get_username_from_api_key(api_key: str): +def _get_username_from_api_key(api_key: str): response = requests.get(url=f"{_LIGHTNING_CLOUD_URL}/v1/auth/user", auth=HTTPBasicAuth("lightning", api_key)) if response.status_code != 200: raise ConnectionRefusedError( @@ -48,7 +48,7 @@ def _check_browser_runnable(): return True -def authenticate(inp_api_key: str = ""): +def _authenticate(inp_api_key: str = ""): # TODO: we have headless login now, # so it could be reasonable to just point to that if browser can't be opened / user can't be authed if not inp_api_key: diff --git a/src/lightning/store/save.py b/src/lightning/store/save.py index 88c9e31345fdb..0c5948c7e9b37 100644 --- a/src/lightning/store/save.py +++ b/src/lightning/store/save.py @@ -290,7 +290,7 @@ def _download_and_extract_data_to(output_dir: str, download_url: str, progress_b _common_clean_up(output_dir) -def get_linked_output_dir(src_dir: str): +def _get_linked_output_dir(src_dir: str): # The last sub-folder will be our version version_folder_name = PurePath(src_dir).parts[-1] diff --git a/src/lightning/store/utils.py b/src/lightning/store/utils.py index f6e609aab6b98..1f349d809f553 100644 --- a/src/lightning/store/utils.py +++ b/src/lightning/store/utils.py @@ -45,13 +45,13 @@ def _split_name(name: str, version: str, l_stage: stage): return username, model_name, version -def split_name(name: str, version: str, l_stage: stage): +def _split_name(name: str, version: str, l_stage: stage): username, model_name, version = _split_name(name, version, l_stage) return username, model_name, version -def get_model_data(name: str, version: str): +def _get_model_data(name: str, version: str): username, model_name, version = split_name(name, version, stage.LOAD) if not os.path.exists(_LIGHTNING_STORAGE_FILE): From 01e3e3048354c77d030f67e003392837cf85c48d Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 26 Jan 2023 15:03:05 +0100 Subject: [PATCH 39/42] protected --- src/lightning/store/authentication.py | 4 ++-- src/lightning/store/cloud_api.py | 16 ++++++++-------- src/lightning/store/utils.py | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/lightning/store/authentication.py b/src/lightning/store/authentication.py index 49701e04aa214..b5cef52894225 100644 --- a/src/lightning/store/authentication.py +++ b/src/lightning/store/authentication.py @@ -58,7 +58,7 @@ def _authenticate(inp_api_key: str = ""): " In order to run the commands on this system, we suggest passing the `api_key`" " after logging into https://lightning.ai." ) - username, inp_api_key = get_user_details() + username, inp_api_key = _get_user_details() else: - username = get_username_from_api_key(inp_api_key) + username = _get_username_from_api_key(inp_api_key) return username, inp_api_key diff --git a/src/lightning/store/cloud_api.py b/src/lightning/store/cloud_api.py index 8641988682270..ca12fbed8cfd5 100644 --- a/src/lightning/store/cloud_api.py +++ b/src/lightning/store/cloud_api.py @@ -25,9 +25,10 @@ import lightning as L import pytorch_lightning as PL from lightning.app.core.constants import LIGHTNING_MODELS_PUBLIC_REGISTRY -from lightning.store.authentication import authenticate +from lightning.store.authentication import _authenticate from lightning.store.save import ( _download_and_extract_data_to, + _get_linked_output_dir, _LIGHTNING_STORAGE_DIR, _LIGHTNING_STORAGE_FILE, _save_checkpoint_from_path, @@ -38,9 +39,8 @@ _save_requirements_file, _submit_data_to_url, _write_and_save_requirements, - get_linked_output_dir, ) -from lightning.store.utils import get_model_data, split_name, stage +from lightning.store.utils import _get_model_data, _split_name, stage logging.basicConfig(level=logging.INFO) @@ -118,8 +118,8 @@ def to_lightning_cloud( ) version = version or "latest" - _, model_name, _ = split_name(name, version=version, l_stage=stage.UPLOAD) - username_from_api_key, api_key = authenticate(api_key) + _, model_name, _ = _split_name(name, version=version, l_stage=stage.UPLOAD) + username_from_api_key, api_key = _authenticate(api_key) # name = f"{username_from_api_key}/{model_name}:{version}" @@ -256,13 +256,13 @@ def download_from_lightning_cloud( Show progress on download. """ version = version or "latest" - username, model_name, version = split_name(name, version=version, l_stage=stage.DOWNLOAD) + username, model_name, version = _split_name(name, version=version, l_stage=stage.DOWNLOAD) linked_output_dir = "" if not output_dir: output_dir = _LIGHTNING_STORAGE_DIR output_dir = os.path.join(output_dir, username, model_name, version) - linked_output_dir = get_linked_output_dir(output_dir) + linked_output_dir = _get_linked_output_dir(output_dir) else: output_dir = os.path.abspath(output_dir) @@ -353,7 +353,7 @@ def load_from_lightning_cloud( if os.path.exists(_LIGHTNING_STORAGE_FILE): version = version or "latest" - model_data = get_model_data(name, version) + model_data = _get_model_data(name, version) output_dir = model_data["output_dir"] linked_output_dir = model_data["linked_output_dir"] meta_data = model_data["metadata"] diff --git a/src/lightning/store/utils.py b/src/lightning/store/utils.py index 1f349d809f553..8618877d81862 100644 --- a/src/lightning/store/utils.py +++ b/src/lightning/store/utils.py @@ -52,7 +52,7 @@ def _split_name(name: str, version: str, l_stage: stage): def _get_model_data(name: str, version: str): - username, model_name, version = split_name(name, version, stage.LOAD) + username, model_name, version = _split_name(name, version, stage.LOAD) if not os.path.exists(_LIGHTNING_STORAGE_FILE): raise NotADirectoryError( From c5dce15b010fe44fd5c4582c193ce79fe8d379d4 Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 26 Jan 2023 15:38:31 +0100 Subject: [PATCH 40/42] fix --- src/lightning/store/utils.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/lightning/store/utils.py b/src/lightning/store/utils.py index 8618877d81862..df7c877ef8914 100644 --- a/src/lightning/store/utils.py +++ b/src/lightning/store/utils.py @@ -15,6 +15,7 @@ import json import os from enum import Enum +from typing import Tuple from lightning.store.save import _LIGHTNING_STORAGE_FILE @@ -35,7 +36,7 @@ def _check_version(version: str) -> bool: return True -def _split_name(name: str, version: str, l_stage: stage): +def _split_name(name: str, version: str, l_stage: stage) -> Tuple[str, str, str]: if l_stage == stage.UPLOAD: username = "" model_name = name @@ -45,12 +46,6 @@ def _split_name(name: str, version: str, l_stage: stage): return username, model_name, version -def _split_name(name: str, version: str, l_stage: stage): - username, model_name, version = _split_name(name, version, l_stage) - - return username, model_name, version - - def _get_model_data(name: str, version: str): username, model_name, version = _split_name(name, version, stage.LOAD) From 017f2713bc1b7b0b183c3c2155b97df19c76a97f Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 26 Jan 2023 17:28:59 +0100 Subject: [PATCH 41/42] renaming --- src/lightning/store/README.md | 59 +++++++++++++++++--------- src/lightning/store/__init__.py | 4 +- src/lightning/store/cloud_api.py | 14 +++--- src/lightning/store/save.py | 4 +- tests/tests_cloud/test_model.py | 26 ++++++------ tests/tests_cloud/test_requirements.py | 10 ++--- tests/tests_cloud/test_source_code.py | 22 +++++----- tests/tests_cloud/test_versioning.py | 10 ++--- 8 files changed, 86 insertions(+), 63 deletions(-) diff --git a/src/lightning/store/README.md b/src/lightning/store/README.md index 522f4aef809a0..5f3ac7150c570 100644 --- a/src/lightning/store/README.md +++ b/src/lightning/store/README.md @@ -1,10 +1,10 @@ ## Getting Started - Login to lightning.ai (_optional_) \<-- takes less than a minute. ⏩ -- Store your models on the cloud \<-- simple call: `to_lightning_cloud(...)`. 🗳️ +- Store your models on the cloud \<-- simple call: `upload_to_cloud(...)`. 🗳️ - Share it with your friends \<-- just share the "username/model_name" (and version if required) format. :handshake: -- They download using a simple call: `download_from_lightning_cloud("username/model_name", version="your_version")`. :wink: -- They load your cool model. `load_from_lightning_cloud("username/model_name", version="your_version")`. :tada: +- They download using a simple call: `download_from_cloud("username/model_name", version="your_version")`. :wink: +- They load your cool model. `load_model("username/model_name", version="your_version")`. :tada: - Lightning :zap: fast, isn't it?. :heart: ## Usage @@ -12,7 +12,7 @@ **Storing to the cloud** ```python -from lightning.store import to_lightning_cloud +import lightning as L from sample.model import LitAutoEncoder, Encoder, Decoder # Initialize your model here @@ -21,10 +21,10 @@ autoencoder = LitAutoEncoder(Encoder(), Decoder()) # Pass the model object: # No need to pass the username (we'll deduce ourselves), just pass the model name you want as the first argument (with an optional version): # format: `model_name:version` (version can either be latest or combination of digits and full-stops: 1.0.0 for example) -to_lightning_cloud("unique_model_mnist", model=autoencoder, source_code_path="sample") +L.store.upload_to_cloud("unique_model_mnist", model=autoencoder, source_code_path="sample") # version: -to_lightning_cloud( +L.store.upload_to_cloud( "unique_model_mnist", version="1.0.0", model=autoencoder, @@ -32,34 +32,49 @@ to_lightning_cloud( ) # OR: (this will save the file which has the model defined) -to_lightning_cloud("krshrimali/unique_model_mnist", model=autoencoder) +L.store.upload_to_cloud("krshrimali/unique_model_mnist", model=autoencoder) ``` You can also pass the checkpoint path: `to_lightning_cloud("model_name", version="latest", checkpoint_path=...)`. **Downloading from the cloud** +At first, you need to download the model to your local machine. + ```python -from lightning.store import download_from_lightning_cloud +import lightning as L -download_from_lightning_cloud("krshrimali/unique_model_mnist", output_dir="your_output_dir") -# OR: (default to lightning_model_storage $HOME/.lightning/lightning_model_store/username//version_/ folder) -download_from_lightning_cloud("krshrimali/unique_model_mnist") +L.store.download_from_cloud( + "krshrimali/unique_model_mnist", + output_dir="your_output_dir", +) +# OR: (default to model_storage +# $HOME +# |- .lightning +# | |- model_store +# | | |- username +# | | | |- +# | | | | |- version_ +# folder) +L.store.download_from_cloud("krshrimali/unique_model_mnist") ``` **Loading model** +Then you can load the model to your program. + ```python -from lightning.store import load_from_lightning_cloud +import lightning as L # from ..version_. import LitAutoEncoder, Encoder, Decoder -model = load_from_lightning_cloud( - "/>", version="version" -) # version is optional (defaults to latest) +model = L.store.load_model("/>", version="version") # version is optional (defaults to latest) # OR: load weights or checkpoint (if they were uploaded) -load_from_lightning_cloud( - "/", version="version", load_weights=True / False, load_checkpoint=True / False +L.store.load_model( + "/", + version="version", + load_weights=True|False, + load_checkpoint=True|False ) print(model) ``` @@ -67,15 +82,21 @@ print(model) **Loading model weights** ```python -from lightning.store import load_from_lightning_cloud +import lightning as L +from sample.model import LitAutoEncoder, Encoder, Decoder # If you had passed an `output_dir=...` to download_from_lightning_cloud(...), then you can just do: # from output_dir. import LitAutoEncoder, Encoder, Decoder model = LitAutoEncoder(Encoder(), Decoder()) -model = load_from_lightning_cloud(load_weights=True, model=model) +model = L.store.load_model(load_weights=True, model=model) print("State dict: ", model.state_dict()) ``` Loading checkpoint is similar, just do: `load_checkpoint=True`. + +## Known limitations + +- missing web UI for user to brows his uploads +- missing CLI/API to list and delete uploaded models diff --git a/src/lightning/store/__init__.py b/src/lightning/store/__init__.py index 31ce7ade8ef96..645891e717534 100644 --- a/src/lightning/store/__init__.py +++ b/src/lightning/store/__init__.py @@ -1,3 +1,3 @@ -from lightning.store.cloud_api import download_from_lightning_cloud, load_from_lightning_cloud, to_lightning_cloud +from lightning.store.cloud_api import download_from_cloud, load_model, upload_to_cloud -__all__ = ["download_from_lightning_cloud", "load_from_lightning_cloud", "to_lightning_cloud"] +__all__ = ["download_from_cloud", "load_model", "upload_to_cloud"] diff --git a/src/lightning/store/cloud_api.py b/src/lightning/store/cloud_api.py index ca12fbed8cfd5..6b31a44d7ce1a 100644 --- a/src/lightning/store/cloud_api.py +++ b/src/lightning/store/cloud_api.py @@ -17,7 +17,7 @@ import os import sys import tempfile -from typing import Any, List, Optional +from typing import Any, List, Optional, Union import requests import torch @@ -45,7 +45,7 @@ logging.basicConfig(level=logging.INFO) -def to_lightning_cloud( +def upload_to_cloud( name: str, version: str = "latest", model=None, @@ -235,7 +235,7 @@ def _load_checkpoint(model, stored, output_dir, *args, **kwargs): return ckpt -def download_from_lightning_cloud( +def download_from_cloud( name: str, version: str = "latest", output_dir: str = "", @@ -250,7 +250,7 @@ def download_from_lightning_cloud( The version of the model to be uploaded. If not provided, default will be latest (not overridden). output_dir: The target directory, where the model and other data will be stored. If not passed, - the data will be stored in `$HOME/.lightning/lightning_model_store///`. + the data will be stored in `$HOME/.lightning/model_store///`. (`version` defaults to `latest`) progress_bar: Show progress on download. @@ -324,12 +324,12 @@ def _validate_output_dir(folder: str) -> None: ) -def load_from_lightning_cloud( +def load_model( name: str, version: str = "latest", load_weights: bool = False, load_checkpoint: bool = False, - model=None, + model: Union[PL.LightningModule, L.LightningModule, None] = None, *args, **kwargs, ): @@ -344,6 +344,8 @@ def load_from_lightning_cloud( Loads only weights if this is set to `True`. Needs `model` to be passed in order to load the weights. load_checkpoint: Loads checkpoint if this is set to `True`. Only a `LightningModule` model is supported for this feature. + model: + Model class to be used. """ if load_weights and load_checkpoint: raise ValueError( diff --git a/src/lightning/store/save.py b/src/lightning/store/save.py index 0c5948c7e9b37..e179cfc05ca04 100644 --- a/src/lightning/store/save.py +++ b/src/lightning/store/save.py @@ -31,8 +31,8 @@ logging.basicConfig(level=logging.INFO) _LIGHTNING_DIR = os.path.join(os.path.expanduser("~"), ".lightning") -_LIGHTNING_STORAGE_FILE = os.path.join(_LIGHTNING_DIR, ".lightning_model_storage") -_LIGHTNING_STORAGE_DIR = os.path.join(_LIGHTNING_DIR, "lightning_model_store") +_LIGHTNING_STORAGE_FILE = os.path.join(_LIGHTNING_DIR, ".model_storage") +_LIGHTNING_STORAGE_DIR = os.path.join(_LIGHTNING_DIR, "model_store") def _check_id(id: str) -> str: diff --git a/tests/tests_cloud/test_model.py b/tests/tests_cloud/test_model.py index c9ad3b9e5ecc3..6eae880b3758e 100644 --- a/tests/tests_cloud/test_model.py +++ b/tests/tests_cloud/test_model.py @@ -4,7 +4,7 @@ from tests_cloud.helpers import cleanup import pytorch_lightning as pl -from lightning.store import download_from_lightning_cloud, load_from_lightning_cloud, to_lightning_cloud +from lightning.store import download_from_cloud, load_model, upload_to_cloud from lightning.store.save import _LIGHTNING_STORAGE_DIR from pytorch_lightning.demos.boring_classes import BoringModel @@ -12,24 +12,24 @@ def test_model(model_name: str = "boring_model", version: str = "latest"): cleanup() - to_lightning_cloud(model_name, model=BoringModel(), api_key=_API_KEY, project_id=_PROJECT_ID) + upload_to_cloud(model_name, model=BoringModel(), api_key=_API_KEY, project_id=_PROJECT_ID) - download_from_lightning_cloud(f"{_USERNAME}/{model_name}") + download_from_cloud(f"{_USERNAME}/{model_name}") assert os.path.isdir(os.path.join(_LIGHTNING_STORAGE_DIR, _USERNAME, model_name, version)) - model = load_from_lightning_cloud(f"{_USERNAME}/{model_name}") + model = load_model(f"{_USERNAME}/{model_name}") assert model is not None def test_model_without_progress_bar(model_name: str = "boring_model", version: str = "latest"): cleanup() - to_lightning_cloud(model_name, model=BoringModel(), api_key=_API_KEY, project_id=_PROJECT_ID, progress_bar=False) + upload_to_cloud(model_name, model=BoringModel(), api_key=_API_KEY, project_id=_PROJECT_ID, progress_bar=False) - download_from_lightning_cloud(f"{_USERNAME}/{model_name}", progress_bar=False) + download_from_cloud(f"{_USERNAME}/{model_name}", progress_bar=False) assert os.path.isdir(os.path.join(_LIGHTNING_STORAGE_DIR, _USERNAME, model_name, version)) - model = load_from_lightning_cloud(f"{_USERNAME}/{model_name}") + model = load_model(f"{_USERNAME}/{model_name}") assert model is not None @@ -39,12 +39,12 @@ def test_only_weights(model_name: str = "boring_model_only_weights", version: st model = BoringModel() trainer = pl.Trainer(fast_dev_run=True) trainer.fit(model) - to_lightning_cloud(model_name, model=model, weights_only=True, api_key=_API_KEY, project_id=_PROJECT_ID) + upload_to_cloud(model_name, model=model, weights_only=True, api_key=_API_KEY, project_id=_PROJECT_ID) - download_from_lightning_cloud(f"{_USERNAME}/{model_name}") + download_from_cloud(f"{_USERNAME}/{model_name}") assert os.path.isdir(os.path.join(_LIGHTNING_STORAGE_DIR, _USERNAME, model_name, version)) - model_with_weights = load_from_lightning_cloud(f"{_USERNAME}/{model_name}", load_weights=True, model=model) + model_with_weights = load_model(f"{_USERNAME}/{model_name}", load_weights=True, model=model) assert model_with_weights is not None assert model_with_weights.state_dict() is not None @@ -56,10 +56,10 @@ def test_checkpoint_path(model_name: str = "boring_model_only_checkpoint_path", trainer = pl.Trainer(fast_dev_run=True) trainer.fit(model) trainer.save_checkpoint("tmp.ckpt") - to_lightning_cloud(model_name, checkpoint_path="tmp.ckpt", api_key=_API_KEY, project_id=_PROJECT_ID) + upload_to_cloud(model_name, checkpoint_path="tmp.ckpt", api_key=_API_KEY, project_id=_PROJECT_ID) - download_from_lightning_cloud(f"{_USERNAME}/{model_name}") + download_from_cloud(f"{_USERNAME}/{model_name}") assert os.path.isdir(os.path.join(_LIGHTNING_STORAGE_DIR, _USERNAME, model_name, version)) - ckpt = load_from_lightning_cloud(f"{_USERNAME}/{model_name}", load_checkpoint=True, model=model) + ckpt = load_model(f"{_USERNAME}/{model_name}", load_checkpoint=True, model=model) assert ckpt is not None diff --git a/tests/tests_cloud/test_requirements.py b/tests/tests_cloud/test_requirements.py index 92f162ebb8047..fc8d2685ed1c6 100644 --- a/tests/tests_cloud/test_requirements.py +++ b/tests/tests_cloud/test_requirements.py @@ -3,7 +3,7 @@ from tests_cloud import _API_KEY, _PROJECT_ID, _PROJECT_ROOT, _USERNAME from tests_cloud.helpers import cleanup -from lightning.store import download_from_lightning_cloud, to_lightning_cloud +from lightning.store import download_from_cloud, upload_to_cloud from lightning.store.save import _LIGHTNING_STORAGE_DIR from pytorch_lightning.demos.boring_classes import BoringModel @@ -13,7 +13,7 @@ def test_requirements_as_a_file(version: str = "latest", model_name: str = "bori requirements_file_path = os.path.join(_PROJECT_ROOT, "requirements", "app", "base.txt") - to_lightning_cloud( + upload_to_cloud( model_name, version=version, model=BoringModel(), @@ -22,7 +22,7 @@ def test_requirements_as_a_file(version: str = "latest", model_name: str = "bori project_id=_PROJECT_ID, ) - download_from_lightning_cloud(f"{_USERNAME}/{model_name}") + download_from_cloud(f"{_USERNAME}/{model_name}") req_folder_path = os.path.join(_LIGHTNING_STORAGE_DIR, _USERNAME, model_name, version) assert os.path.isdir(req_folder_path), "missing: %s" % req_folder_path @@ -34,7 +34,7 @@ def test_requirements_as_a_list(version: str = "1.0.0", model_name: str = "borin requirements_list = ["pytorch_lightning==1.7.7", "lightning"] - to_lightning_cloud( + upload_to_cloud( model_name, version=version, model=BoringModel(), @@ -43,7 +43,7 @@ def test_requirements_as_a_list(version: str = "1.0.0", model_name: str = "borin project_id=_PROJECT_ID, ) - download_from_lightning_cloud(f"{_USERNAME}/{model_name}", version=version) + download_from_cloud(f"{_USERNAME}/{model_name}", version=version) req_folder_path = os.path.join(_LIGHTNING_STORAGE_DIR, _USERNAME, model_name, version) assert os.path.isdir(req_folder_path), "missing: %s" % req_folder_path diff --git a/tests/tests_cloud/test_source_code.py b/tests/tests_cloud/test_source_code.py index f7dec36389064..11ea89775e484 100644 --- a/tests/tests_cloud/test_source_code.py +++ b/tests/tests_cloud/test_source_code.py @@ -5,7 +5,7 @@ from tests_cloud import _API_KEY, _PROJECT_ID, _PROJECT_ROOT, _TEST_ROOT, _USERNAME from tests_cloud.helpers import cleanup -from lightning.store import download_from_lightning_cloud, to_lightning_cloud +from lightning.store import download_from_cloud, upload_to_cloud from lightning.store.save import _LIGHTNING_STORAGE_DIR from pytorch_lightning.demos.boring_classes import BoringModel @@ -13,9 +13,9 @@ def test_source_code_implicit(model_name: str = "model_test_source_code_implicit"): cleanup() - to_lightning_cloud(model_name, model=BoringModel(), api_key=_API_KEY, project_id=_PROJECT_ID) + upload_to_cloud(model_name, model=BoringModel(), api_key=_API_KEY, project_id=_PROJECT_ID) - download_from_lightning_cloud(f"{_USERNAME}/{model_name}") + download_from_cloud(f"{_USERNAME}/{model_name}") assert os.path.isfile( os.path.join( _LIGHTNING_STORAGE_DIR, @@ -30,9 +30,9 @@ def test_source_code_implicit(model_name: str = "model_test_source_code_implicit def test_source_code_saving_disabled(model_name: str = "model_test_source_code_dont_save"): cleanup() - to_lightning_cloud(model_name, model=BoringModel(), api_key=_API_KEY, project_id=_PROJECT_ID, save_code=False) + upload_to_cloud(model_name, model=BoringModel(), api_key=_API_KEY, project_id=_PROJECT_ID, save_code=False) - download_from_lightning_cloud(f"{_USERNAME}/{model_name}") + download_from_cloud(f"{_USERNAME}/{model_name}") assert not os.path.isfile( os.path.join( _LIGHTNING_STORAGE_DIR, @@ -48,11 +48,11 @@ def test_source_code_explicit_relative_folder(model_name: str = "model_test_sour cleanup() dir_upload_path = _TEST_ROOT - to_lightning_cloud( + upload_to_cloud( model_name, model=BoringModel(), source_code_path=dir_upload_path, api_key=_API_KEY, project_id=_PROJECT_ID ) - download_from_lightning_cloud(f"{_USERNAME}/{model_name}") + download_from_cloud(f"{_USERNAME}/{model_name}") assert os.path.isdir( os.path.join( @@ -66,11 +66,11 @@ def test_source_code_explicit_absolute_folder(model_name: str = "model_test_sour with tempfile.TemporaryDirectory() as tmpdir: dir_upload_path = os.path.abspath(tmpdir) - to_lightning_cloud( + upload_to_cloud( model_name, model=BoringModel(), source_code_path=dir_upload_path, api_key=_API_KEY, project_id=_PROJECT_ID ) - download_from_lightning_cloud(f"{_USERNAME}/{model_name}") + download_from_cloud(f"{_USERNAME}/{model_name}") assert os.path.isdir( os.path.join( @@ -83,11 +83,11 @@ def test_source_code_explicit_file(model_name: str = "model_test_source_code_exp cleanup() file_name = os.path.join(_PROJECT_ROOT, "setup.py") - to_lightning_cloud( + upload_to_cloud( model_name, model=BoringModel(), source_code_path=file_name, api_key=_API_KEY, project_id=_PROJECT_ID ) - download_from_lightning_cloud(f"{_USERNAME}/{model_name}") + download_from_cloud(f"{_USERNAME}/{model_name}") assert os.path.isfile( os.path.join( diff --git a/tests/tests_cloud/test_versioning.py b/tests/tests_cloud/test_versioning.py index 8c52ec3159fcb..373d0a7500504 100644 --- a/tests/tests_cloud/test_versioning.py +++ b/tests/tests_cloud/test_versioning.py @@ -5,7 +5,7 @@ from tests_cloud import _API_KEY, _PROJECT_ID, _USERNAME from tests_cloud.helpers import cleanup -from lightning.store.cloud_api import download_from_lightning_cloud, to_lightning_cloud +from lightning.store.cloud_api import download_from_cloud, upload_to_cloud from lightning.store.save import _LIGHTNING_STORAGE_DIR from pytorch_lightning.demos.boring_classes import BoringModel @@ -33,8 +33,8 @@ def assert_download_successful(username, model_name, version): def test_versioning_valid_case(case, expected_case, model_name: str = "boring_model_versioning"): cleanup() - to_lightning_cloud(model_name, version=case, model=BoringModel(), api_key=_API_KEY, project_id=_PROJECT_ID) - download_from_lightning_cloud(f"{_USERNAME}/{model_name}", version=case) + upload_to_cloud(model_name, version=case, model=BoringModel(), api_key=_API_KEY, project_id=_PROJECT_ID) + download_from_cloud(f"{_USERNAME}/{model_name}", version=case) assert_download_successful(_USERNAME, model_name, expected_case) @@ -54,8 +54,8 @@ def test_versioning_invalid_case(case, model_name: str = "boring_model_versionin cleanup() with pytest.raises(ConnectionRefusedError): - to_lightning_cloud(model_name, version=case, model=BoringModel(), api_key=_API_KEY, project_id=_PROJECT_ID) + upload_to_cloud(model_name, version=case, model=BoringModel(), api_key=_API_KEY, project_id=_PROJECT_ID) error = OSError if case == "*" and platform.system() == "Windows" else ConnectionRefusedError with pytest.raises(error): - download_from_lightning_cloud(f"{_USERNAME}/{model_name}", version=case) + download_from_cloud(f"{_USERNAME}/{model_name}", version=case) From 5e1f7c6e216e7e5e750dbccc174b3f174e92f473 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 26 Jan 2023 16:30:06 +0000 Subject: [PATCH 42/42] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/lightning/store/README.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/lightning/store/README.md b/src/lightning/store/README.md index 5f3ac7150c570..85ea84335e648 100644 --- a/src/lightning/store/README.md +++ b/src/lightning/store/README.md @@ -48,7 +48,7 @@ L.store.download_from_cloud( "krshrimali/unique_model_mnist", output_dir="your_output_dir", ) -# OR: (default to model_storage +# OR: (default to model_storage # $HOME # |- .lightning # | |- model_store @@ -71,10 +71,7 @@ model = L.store.load_model("/>", version="version") # ver # OR: load weights or checkpoint (if they were uploaded) L.store.load_model( - "/", - version="version", - load_weights=True|False, - load_checkpoint=True|False + "/", version="version", load_weights=True | False, load_checkpoint=True | False ) print(model) ```