From ad9fffd610bbadb16ea7d4bf78bebb3a2ad22d47 Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Thu, 9 Nov 2023 10:52:29 +0000 Subject: [PATCH 1/6] refactor: remove default raycluster image --- src/codeflare_sdk/cluster/config.py | 2 +- tests/unit_test_support.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/codeflare_sdk/cluster/config.py b/src/codeflare_sdk/cluster/config.py index 192097646..0311d0e32 100644 --- a/src/codeflare_sdk/cluster/config.py +++ b/src/codeflare_sdk/cluster/config.py @@ -48,7 +48,7 @@ class ClusterConfiguration: instascale: bool = False mcad: bool = True envs: dict = field(default_factory=dict) - image: str = "quay.io/project-codeflare/ray:latest-py39-cu118" + image: str = "" local_interactive: bool = False image_pull_secrets: list = field(default_factory=list) dispatch_priority: str = None diff --git a/tests/unit_test_support.py b/tests/unit_test_support.py index fea4ceafd..6d61130a8 100644 --- a/tests/unit_test_support.py +++ b/tests/unit_test_support.py @@ -47,6 +47,7 @@ def createClusterConfig(): machine_types=["cpu.small", "gpu.large"], image_pull_secrets=["unit-test-pull-secret"], ingress_domain="apps.cluster.awsroute.org", + image="quay.io/project-codeflare/ray:latest-py39-cu118" ) return config From 677bca5783fbb0675dfd5c24bcd0bf0653a36ad0 Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Thu, 9 Nov 2023 10:53:06 +0000 Subject: [PATCH 2/6] refactor: update demo notebooks with raycluster images and note --- .../additional-demos/hf_interactive.ipynb | 5 +++- .../additional-demos/local_interactive.ipynb | 23 ++++++++++++++++++- demo-notebooks/guided-demos/0_basic_ray.ipynb | 9 +++++--- .../guided-demos/1_basic_instascale.ipynb | 6 ++++- .../guided-demos/2_basic_jobs.ipynb | 6 ++++- .../guided-demos/3_basic_interactive.ipynb | 6 ++++- demo-notebooks/guided-demos/4_gpt.ipynb | 10 ++++++++ .../notebook-ex-outputs/0_basic_ray.ipynb | 7 ++++-- .../1_basic_instascale.ipynb | 6 ++++- .../notebook-ex-outputs/2_basic_jobs.ipynb | 6 ++++- .../3_basic_interactive.ipynb | 6 ++++- .../notebook-ex-outputs/4_gpt.ipynb | 11 +++++++++ .../preview_nbs/0_basic_ray.ipynb | 7 ++++-- .../preview_nbs/1_basic_instascale.ipynb | 6 ++++- .../preview_nbs/2_basic_jobs.ipynb | 6 ++++- .../preview_nbs/3_basic_interactive.ipynb | 6 ++++- .../guided-demos/preview_nbs/4_gpt.ipynb | 10 ++++++++ 17 files changed, 118 insertions(+), 18 deletions(-) diff --git a/demo-notebooks/additional-demos/hf_interactive.ipynb b/demo-notebooks/additional-demos/hf_interactive.ipynb index a6780a8f8..45e9653fc 100644 --- a/demo-notebooks/additional-demos/hf_interactive.ipynb +++ b/demo-notebooks/additional-demos/hf_interactive.ipynb @@ -69,7 +69,10 @@ "id": "bc27f84c", "metadata": {}, "source": [ - "Here, we want to define our cluster by specifying the resources we require for our batch workload. Below, we define our cluster object (which generates a corresponding AppWrapper)." + "Here, we want to define our cluster by specifying the resources we require for our batch workload. Below, we define our cluster object (which generates a corresponding AppWrapper).\n", + "\n", + "NOTE: We must specify the `image` which will be used in our RayCluster, we recommend you bring your own image which suits your purposes. \n", + "The example here is a community image." ] }, { diff --git a/demo-notebooks/additional-demos/local_interactive.ipynb b/demo-notebooks/additional-demos/local_interactive.ipynb index 7533db3d8..b6773ee1a 100644 --- a/demo-notebooks/additional-demos/local_interactive.ipynb +++ b/demo-notebooks/additional-demos/local_interactive.ipynb @@ -30,6 +30,16 @@ "auth.login()" ] }, + { + "cell_type": "markdown", + "id": "18de2d65", + "metadata": {}, + "source": [ + "\n", + "NOTE: We must specify the `image` which will be used in our RayCluster, we recommend you bring your own image which suits your purposes. \n", + "The example here is a community image." + ] + }, { "cell_type": "code", "execution_count": null, @@ -44,7 +54,18 @@ "cluster_name = \"hfgputest-1\"\n", "local_interactive = True\n", "\n", - "cluster = Cluster(ClusterConfiguration(local_interactive=local_interactive, namespace=namespace, name=cluster_name, num_workers=1, min_cpus=1, max_cpus=1, min_memory=4, max_memory=4, num_gpus=0, instascale=False, machine_types=[\"m5.xlarge\", \"p3.8xlarge\"]))" + "cluster = Cluster(ClusterConfiguration(local_interactive=local_interactive,\n", + " namespace=namespace,\n", + " name=cluster_name,\n", + " num_workers=1,\n", + " min_cpus=1,\n", + " max_cpus=1,\n", + " min_memory=4,\n", + " max_memory=4,\n", + " num_gpus=0,\n", + " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n", + " instascale=False,\n", + " machine_types=[\"m5.xlarge\", \"p3.8xlarge\"]))" ] }, { diff --git a/demo-notebooks/guided-demos/0_basic_ray.ipynb b/demo-notebooks/guided-demos/0_basic_ray.ipynb index d37b7d7d0..c905d1756 100644 --- a/demo-notebooks/guided-demos/0_basic_ray.ipynb +++ b/demo-notebooks/guided-demos/0_basic_ray.ipynb @@ -46,7 +46,10 @@ "id": "bc27f84c", "metadata": {}, "source": [ - "Here, we want to define our cluster by specifying the resources we require for our batch workload. Below, we define our cluster object (which generates a corresponding AppWrapper)." + "Here, we want to define our cluster by specifying the resources we require for our batch workload. Below, we define our cluster object (which generates a corresponding AppWrapper).\n", + "\n", + "NOTE: We must specify the `image` which will be used in our RayCluster, we recommend you bring your own image which suits your purposes. \n", + "The example here is a community image." ] }, { @@ -66,7 +69,7 @@ " min_memory=4,\n", " max_memory=4,\n", " num_gpus=0,\n", - " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\", #current default\n", + " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n", " instascale=False\n", "))" ] @@ -191,7 +194,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.13" + "version": "3.9.18" }, "vscode": { "interpreter": { diff --git a/demo-notebooks/guided-demos/1_basic_instascale.ipynb b/demo-notebooks/guided-demos/1_basic_instascale.ipynb index d0faf5b9b..f1795382e 100644 --- a/demo-notebooks/guided-demos/1_basic_instascale.ipynb +++ b/demo-notebooks/guided-demos/1_basic_instascale.ipynb @@ -43,7 +43,10 @@ "id": "bc27f84c", "metadata": {}, "source": [ - "This time, we are working in a cloud environment, and our OpenShift cluster does not have the resources needed for our desired workloads. We will use InstaScale to dynamically scale-up guaranteed resources based on our request (that will also automatically scale-down when we are finished working):" + "This time, we are working in a cloud environment, and our OpenShift cluster does not have the resources needed for our desired workloads. We will use InstaScale to dynamically scale-up guaranteed resources based on our request (that will also automatically scale-down when we are finished working):\n", + "\n", + "NOTE: We must specify the `image` which will be used in our RayCluster, we recommend you bring your own image which suits your purposes. \n", + "The example here is a community image." ] }, { @@ -63,6 +66,7 @@ " min_memory=8,\n", " max_memory=8,\n", " num_gpus=1,\n", + " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n", " instascale=True, # InstaScale now enabled, will scale OCP cluster to guarantee resource request\n", " machine_types=[\"m5.xlarge\", \"g4dn.xlarge\"] # Head, worker AWS machine types desired\n", "))" diff --git a/demo-notebooks/guided-demos/2_basic_jobs.ipynb b/demo-notebooks/guided-demos/2_basic_jobs.ipynb index da74f9e54..5d862c03a 100644 --- a/demo-notebooks/guided-demos/2_basic_jobs.ipynb +++ b/demo-notebooks/guided-demos/2_basic_jobs.ipynb @@ -43,7 +43,10 @@ "id": "bc27f84c", "metadata": {}, "source": [ - "Let's start by running through the same cluster setup as before:" + "Let's start by running through the same cluster setup as before:\n", + "\n", + "NOTE: We must specify the `image` which will be used in our RayCluster, we recommend you bring your own image which suits your purposes. \n", + "The example here is a community image." ] }, { @@ -63,6 +66,7 @@ " min_memory=4,\n", " max_memory=4,\n", " num_gpus=0,\n", + " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n", " instascale=False\n", "))" ] diff --git a/demo-notebooks/guided-demos/3_basic_interactive.ipynb b/demo-notebooks/guided-demos/3_basic_interactive.ipynb index c8b2b1a0e..bfcb2df3a 100644 --- a/demo-notebooks/guided-demos/3_basic_interactive.ipynb +++ b/demo-notebooks/guided-demos/3_basic_interactive.ipynb @@ -43,7 +43,10 @@ "id": "bc27f84c", "metadata": {}, "source": [ - "Once again, let's start by running through the same cluster setup as before:" + "Once again, let's start by running through the same cluster setup as before:\n", + "\n", + "NOTE: We must specify the `image` which will be used in our RayCluster, we recommend you bring your own image which suits your purposes. \n", + "The example here is a community image." ] }, { @@ -63,6 +66,7 @@ " min_memory=8,\n", " max_memory=8,\n", " num_gpus=1,\n", + " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n", " instascale=True, #<---instascale enabled\n", " machine_types=[\"m5.xlarge\", \"g4dn.xlarge\"]\n", " \n", diff --git a/demo-notebooks/guided-demos/4_gpt.ipynb b/demo-notebooks/guided-demos/4_gpt.ipynb index 0fdcec965..dba03bc4a 100644 --- a/demo-notebooks/guided-demos/4_gpt.ipynb +++ b/demo-notebooks/guided-demos/4_gpt.ipynb @@ -30,6 +30,15 @@ "auth.login()" ] }, + { + "cell_type": "markdown", + "id": "8f4b200f", + "metadata": {}, + "source": [ + "NOTE: We must specify the `image` which will be used in our RayCluster, we recommend you bring your own image which suits your purposes. \n", + "The example here is a community image." + ] + }, { "cell_type": "code", "execution_count": null, @@ -46,6 +55,7 @@ " min_memory=8,\n", " max_memory=8,\n", " num_gpus=1,\n", + " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n", " instascale=True, #<---instascale enabled\n", " machine_types=[\"m5.xlarge\", \"g4dn.xlarge\"],\n", "))" diff --git a/demo-notebooks/guided-demos/notebook-ex-outputs/0_basic_ray.ipynb b/demo-notebooks/guided-demos/notebook-ex-outputs/0_basic_ray.ipynb index 18e5a8405..d07fd4304 100644 --- a/demo-notebooks/guided-demos/notebook-ex-outputs/0_basic_ray.ipynb +++ b/demo-notebooks/guided-demos/notebook-ex-outputs/0_basic_ray.ipynb @@ -46,7 +46,10 @@ "id": "bc27f84c", "metadata": {}, "source": [ - "Here, we want to define our cluster by specifying the resources we require for our batch workload. Below, we define our cluster object (which generates a corresponding AppWrapper)." + "Here, we want to define our cluster by specifying the resources we require for our batch workload. Below, we define our cluster object (which generates a corresponding AppWrapper).\n", + "\n", + "NOTE: We must specify the `image` which will be used in our RayCluster, we recommend you bring your own image which suits your purposes. \n", + "The example here is a community image." ] }, { @@ -74,7 +77,7 @@ " min_memory=4,\n", " max_memory=4,\n", " num_gpus=0,\n", - " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\", #current default\n", + " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n", " instascale=False\n", "))" ] diff --git a/demo-notebooks/guided-demos/notebook-ex-outputs/1_basic_instascale.ipynb b/demo-notebooks/guided-demos/notebook-ex-outputs/1_basic_instascale.ipynb index 97a2b3822..6cfe8143f 100644 --- a/demo-notebooks/guided-demos/notebook-ex-outputs/1_basic_instascale.ipynb +++ b/demo-notebooks/guided-demos/notebook-ex-outputs/1_basic_instascale.ipynb @@ -43,7 +43,10 @@ "id": "bc27f84c", "metadata": {}, "source": [ - "This time, we are working in a cloud environment, and our OpenShift cluster does not have the resources needed for our desired workloads. We will use InstaScale to dynamically scale-up guaranteed resources based on our request (that will also automatically scale-down when we are finished working):" + "This time, we are working in a cloud environment, and our OpenShift cluster does not have the resources needed for our desired workloads. We will use InstaScale to dynamically scale-up guaranteed resources based on our request (that will also automatically scale-down when we are finished working):\n", + "\n", + "NOTE: We must specify the `image` which will be used in our RayCluster, we recommend you bring your own image which suits your purposes. \n", + "The example here is a community image." ] }, { @@ -71,6 +74,7 @@ " min_memory=8,\n", " max_memory=8,\n", " num_gpus=1,\n", + " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n", " instascale=True, # InstaScale now enabled, will scale OCP cluster to guarantee resource request\n", " machine_types=[\"m5.xlarge\", \"g4dn.xlarge\"] # Head, worker AWS machine types desired\n", "))" diff --git a/demo-notebooks/guided-demos/notebook-ex-outputs/2_basic_jobs.ipynb b/demo-notebooks/guided-demos/notebook-ex-outputs/2_basic_jobs.ipynb index 9c6122670..42600a3f0 100644 --- a/demo-notebooks/guided-demos/notebook-ex-outputs/2_basic_jobs.ipynb +++ b/demo-notebooks/guided-demos/notebook-ex-outputs/2_basic_jobs.ipynb @@ -43,7 +43,10 @@ "id": "bc27f84c", "metadata": {}, "source": [ - "Let's start by running through the same cluster setup as before:" + "Let's start by running through the same cluster setup as before:\n", + "\n", + "NOTE: We must specify the `image` which will be used in our RayCluster, we recommend you bring your own image which suits your purposes. \n", + "The example here is a community image." ] }, { @@ -71,6 +74,7 @@ " min_memory=4,\n", " max_memory=4,\n", " num_gpus=0,\n", + " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n", " instascale=False\n", "))" ] diff --git a/demo-notebooks/guided-demos/notebook-ex-outputs/3_basic_interactive.ipynb b/demo-notebooks/guided-demos/notebook-ex-outputs/3_basic_interactive.ipynb index d6799161d..974bd0587 100644 --- a/demo-notebooks/guided-demos/notebook-ex-outputs/3_basic_interactive.ipynb +++ b/demo-notebooks/guided-demos/notebook-ex-outputs/3_basic_interactive.ipynb @@ -43,7 +43,10 @@ "id": "bc27f84c", "metadata": {}, "source": [ - "Once again, let's start by running through the same cluster setup as before:" + "Once again, let's start by running through the same cluster setup as before:\n", + "\n", + "NOTE: We must specify the `image` which will be used in our RayCluster, we recommend you bring your own image which suits your purposes. \n", + "The example here is a community image." ] }, { @@ -71,6 +74,7 @@ " min_memory=8,\n", " max_memory=8,\n", " num_gpus=1,\n", + " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n", " instascale=True, #<---instascale enabled\n", " machine_types=[\"m5.xlarge\", \"g4dn.xlarge\"]\n", " \n", diff --git a/demo-notebooks/guided-demos/notebook-ex-outputs/4_gpt.ipynb b/demo-notebooks/guided-demos/notebook-ex-outputs/4_gpt.ipynb index 704f94f2b..4ed6cc54a 100644 --- a/demo-notebooks/guided-demos/notebook-ex-outputs/4_gpt.ipynb +++ b/demo-notebooks/guided-demos/notebook-ex-outputs/4_gpt.ipynb @@ -30,6 +30,16 @@ "auth.login()" ] }, + { + "cell_type": "markdown", + "id": "b43e8e21", + "metadata": {}, + "source": [ + "\n", + "NOTE: We must specify the `image` which will be used in our RayCluster, we recommend you bring your own image which suits your purposes. \n", + "The example here is a community image." + ] + }, { "cell_type": "code", "execution_count": 2, @@ -54,6 +64,7 @@ " min_memory=8,\n", " max_memory=8,\n", " num_gpus=1,\n", + " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n", " instascale=True, #<---instascale enabled\n", " machine_types=[\"m5.xlarge\", \"g4dn.xlarge\"],\n", "))" diff --git a/demo-notebooks/guided-demos/preview_nbs/0_basic_ray.ipynb b/demo-notebooks/guided-demos/preview_nbs/0_basic_ray.ipynb index d37b7d7d0..be2ca1e56 100644 --- a/demo-notebooks/guided-demos/preview_nbs/0_basic_ray.ipynb +++ b/demo-notebooks/guided-demos/preview_nbs/0_basic_ray.ipynb @@ -46,7 +46,10 @@ "id": "bc27f84c", "metadata": {}, "source": [ - "Here, we want to define our cluster by specifying the resources we require for our batch workload. Below, we define our cluster object (which generates a corresponding AppWrapper)." + "Here, we want to define our cluster by specifying the resources we require for our batch workload. Below, we define our cluster object (which generates a corresponding AppWrapper).\n", + "\n", + "NOTE: We must specify the `image` which will be used in our RayCluster, we recommend you bring your own image which suits your purposes. \n", + "The example here is a community image." ] }, { @@ -66,7 +69,7 @@ " min_memory=4,\n", " max_memory=4,\n", " num_gpus=0,\n", - " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\", #current default\n", + " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n", " instascale=False\n", "))" ] diff --git a/demo-notebooks/guided-demos/preview_nbs/1_basic_instascale.ipynb b/demo-notebooks/guided-demos/preview_nbs/1_basic_instascale.ipynb index d0faf5b9b..f1795382e 100644 --- a/demo-notebooks/guided-demos/preview_nbs/1_basic_instascale.ipynb +++ b/demo-notebooks/guided-demos/preview_nbs/1_basic_instascale.ipynb @@ -43,7 +43,10 @@ "id": "bc27f84c", "metadata": {}, "source": [ - "This time, we are working in a cloud environment, and our OpenShift cluster does not have the resources needed for our desired workloads. We will use InstaScale to dynamically scale-up guaranteed resources based on our request (that will also automatically scale-down when we are finished working):" + "This time, we are working in a cloud environment, and our OpenShift cluster does not have the resources needed for our desired workloads. We will use InstaScale to dynamically scale-up guaranteed resources based on our request (that will also automatically scale-down when we are finished working):\n", + "\n", + "NOTE: We must specify the `image` which will be used in our RayCluster, we recommend you bring your own image which suits your purposes. \n", + "The example here is a community image." ] }, { @@ -63,6 +66,7 @@ " min_memory=8,\n", " max_memory=8,\n", " num_gpus=1,\n", + " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n", " instascale=True, # InstaScale now enabled, will scale OCP cluster to guarantee resource request\n", " machine_types=[\"m5.xlarge\", \"g4dn.xlarge\"] # Head, worker AWS machine types desired\n", "))" diff --git a/demo-notebooks/guided-demos/preview_nbs/2_basic_jobs.ipynb b/demo-notebooks/guided-demos/preview_nbs/2_basic_jobs.ipynb index 4ac4f00af..e3bbbce95 100644 --- a/demo-notebooks/guided-demos/preview_nbs/2_basic_jobs.ipynb +++ b/demo-notebooks/guided-demos/preview_nbs/2_basic_jobs.ipynb @@ -43,7 +43,10 @@ "id": "bc27f84c", "metadata": {}, "source": [ - "Let's start by running through the same cluster setup as before:" + "Let's start by running through the same cluster setup as before:\n", + "\n", + "NOTE: We must specify the `image` which will be used in our RayCluster, we recommend you bring your own image which suits your purposes. \n", + "The example here is a community image." ] }, { @@ -63,6 +66,7 @@ " min_memory=4,\n", " max_memory=4,\n", " num_gpus=0,\n", + " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n", " instascale=False\n", "))" ] diff --git a/demo-notebooks/guided-demos/preview_nbs/3_basic_interactive.ipynb b/demo-notebooks/guided-demos/preview_nbs/3_basic_interactive.ipynb index c8b2b1a0e..bfcb2df3a 100644 --- a/demo-notebooks/guided-demos/preview_nbs/3_basic_interactive.ipynb +++ b/demo-notebooks/guided-demos/preview_nbs/3_basic_interactive.ipynb @@ -43,7 +43,10 @@ "id": "bc27f84c", "metadata": {}, "source": [ - "Once again, let's start by running through the same cluster setup as before:" + "Once again, let's start by running through the same cluster setup as before:\n", + "\n", + "NOTE: We must specify the `image` which will be used in our RayCluster, we recommend you bring your own image which suits your purposes. \n", + "The example here is a community image." ] }, { @@ -63,6 +66,7 @@ " min_memory=8,\n", " max_memory=8,\n", " num_gpus=1,\n", + " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n", " instascale=True, #<---instascale enabled\n", " machine_types=[\"m5.xlarge\", \"g4dn.xlarge\"]\n", " \n", diff --git a/demo-notebooks/guided-demos/preview_nbs/4_gpt.ipynb b/demo-notebooks/guided-demos/preview_nbs/4_gpt.ipynb index 455bb9aa8..77edf5911 100644 --- a/demo-notebooks/guided-demos/preview_nbs/4_gpt.ipynb +++ b/demo-notebooks/guided-demos/preview_nbs/4_gpt.ipynb @@ -30,6 +30,15 @@ "auth.login()" ] }, + { + "cell_type": "markdown", + "id": "5e4e9ee9", + "metadata": {}, + "source": [ + "NOTE: We must specify the `image` which will be used in our RayCluster, we recommend you bring your own image which suits your purposes. \n", + "The example here is a community image." + ] + }, { "cell_type": "code", "execution_count": null, @@ -46,6 +55,7 @@ " min_memory=8,\n", " max_memory=8,\n", " num_gpus=1,\n", + " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n", " instascale=True, #<---instascale enabled\n", " machine_types=[\"m5.xlarge\", \"g4dn.xlarge\"],\n", "))" From 958878a1f0f845b0aad38a046a6612887d9397d3 Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Thu, 9 Nov 2023 14:40:05 +0000 Subject: [PATCH 3/6] refactor: validate image config --- src/codeflare_sdk/cluster/cluster.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/codeflare_sdk/cluster/cluster.py b/src/codeflare_sdk/cluster/cluster.py index a2871000e..a154994d7 100644 --- a/src/codeflare_sdk/cluster/cluster.py +++ b/src/codeflare_sdk/cluster/cluster.py @@ -128,6 +128,17 @@ def evaluate_dispatch_priority(self): print(f"Priority class {priority_class} is not available in the cluster") return None + def validate_image_config(self): + """ + Validates that the image configuration is not empty. + + :param image: The image string to validate + :raises ValueError: If the image is not specified + """ + if self.config.image == "" or self.config.image == None: + raise ValueError("Image must be specified in the ClusterConfiguration") + + def create_app_wrapper(self): """ Called upon cluster object creation, creates an AppWrapper yaml based on @@ -142,6 +153,9 @@ def create_app_wrapper(self): raise TypeError( f"Namespace {self.config.namespace} is of type {type(self.config.namespace)}. Check your Kubernetes Authentication." ) + + # Validate image configuration + self.validate_image_config() # Before attempting to create the cluster AW, let's evaluate the ClusterConfig if self.config.dispatch_priority: From 07bfa3dfe47e51b9803a7ac45c948bacdde0c524 Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Thu, 9 Nov 2023 14:40:39 +0000 Subject: [PATCH 4/6] test: update tests --- tests/unit_test.py | 16 ++++++++++------ tests/unit_test_support.py | 1 + 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/unit_test.py b/tests/unit_test.py index c6b577365..d6eb9ecf0 100644 --- a/tests/unit_test.py +++ b/tests/unit_test.py @@ -21,6 +21,8 @@ import re import uuid +from codeflare_sdk.cluster import cluster + parent = Path(__file__).resolve().parents[1] sys.path.append(str(parent) + "/src") @@ -250,6 +252,7 @@ def test_config_creation(): assert config.dispatch_priority == None assert config.mcad == True assert config.local_interactive == False + def test_cluster_creation(mocker): @@ -259,8 +262,7 @@ def test_cluster_creation(mocker): assert filecmp.cmp( "unit-test-cluster.yaml", f"{parent}/tests/test-case.yaml", shallow=True ) - - + def test_cluster_creation_no_mcad(mocker): mocker.patch( "kubernetes.client.CustomObjectsApi.get_cluster_custom_object", @@ -311,6 +313,7 @@ def test_default_cluster_creation(mocker): ) default_config = ClusterConfiguration( name="unit-test-default-cluster", + image="quay.io/project-codeflare/ray:latest-py39-cu118", ) cluster = Cluster(default_config) @@ -614,6 +617,7 @@ def ingress_retrieval(port, annotations=None): def test_ray_job_wrapping(mocker): cluster = cluster = createClusterWithConfig(mocker) + cluster.config.image = "quay.io/project-codeflare/ray:latest-py39-cu118" mocker.patch( "ray.job_submission.JobSubmissionClient._check_connection_and_version_with_url", return_value="None", @@ -732,7 +736,7 @@ def test_ray_details(mocker, capsys): "codeflare_sdk.cluster.cluster.Cluster.cluster_dashboard_uri", return_value="", ) - cf = Cluster(ClusterConfiguration(name="raytest2", namespace="ns")) + cf = Cluster(ClusterConfiguration(name="raytest2", namespace="ns", image= "quay.io/project-codeflare/ray:latest-py39-cu118")) captured = capsys.readouterr() ray2 = _copy_to_ray(cf) details = cf.details() @@ -1898,7 +1902,7 @@ def test_cluster_status(mocker): head_mem=8, head_gpu=0, ) - cf = Cluster(ClusterConfiguration(name="test", namespace="ns")) + cf = Cluster(ClusterConfiguration(name="test", namespace="ns", image="quay.io/project-codeflare/ray:latest-py39-cu118")) mocker.patch("codeflare_sdk.cluster.cluster._app_wrapper_status", return_value=None) mocker.patch("codeflare_sdk.cluster.cluster._ray_cluster_status", return_value=None) status, ready = cf.status() @@ -1988,7 +1992,7 @@ def test_wait_ready(mocker, capsys): mock_response = mocker.Mock() mock_response.status_code = 200 mocker.patch("requests.get", return_value=mock_response) - cf = Cluster(ClusterConfiguration(name="test", namespace="ns")) + cf = Cluster(ClusterConfiguration(name="test", namespace="ns", image= "quay.io/project-codeflare/ray:latest-py39-cu118")) try: cf.wait_ready(timeout=5) assert 1 == 0 @@ -2653,7 +2657,7 @@ def test_gen_app_wrapper_with_oauth(mocker: MockerFixture): mocker.patch( "codeflare_sdk.utils.generate_yaml.write_user_appwrapper", write_user_appwrapper ) - Cluster(ClusterConfiguration("test_cluster", openshift_oauth=True)) + Cluster(ClusterConfiguration("test_cluster", openshift_oauth=True, image= "quay.io/project-codeflare/ray:latest-py39-cu118")) user_yaml = write_user_appwrapper.call_args.args[0] assert any( container["name"] == "oauth-proxy" diff --git a/tests/unit_test_support.py b/tests/unit_test_support.py index 6d61130a8..4c11c5930 100644 --- a/tests/unit_test_support.py +++ b/tests/unit_test_support.py @@ -59,6 +59,7 @@ def createClusterWithConfig(mocker): return_value={"spec": {"domain": "apps.cluster.awsroute.org"}}, ) cluster = Cluster(createClusterConfig()) + cluster.config.image = "quay.io/project-codeflare/ray:latest-py39-cu118" return cluster From 3a018972d88e6ccce1e56f04f8025a314f5883bf Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Thu, 9 Nov 2023 14:41:28 +0000 Subject: [PATCH 5/6] test: add test which raises error when no image is supplied --- tests/unit_test.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unit_test.py b/tests/unit_test.py index d6eb9ecf0..6f9cf8796 100644 --- a/tests/unit_test.py +++ b/tests/unit_test.py @@ -263,6 +263,18 @@ def test_cluster_creation(mocker): "unit-test-cluster.yaml", f"{parent}/tests/test-case.yaml", shallow=True ) +def test_create_app_wrapper_raises_error_with_no_image(): + config = createClusterConfig() + config.image = "" # Clear the image to test error handling + try: + cluster = Cluster(config) + cluster.create_app_wrapper() + assert False, "Expected ValueError when 'image' is not specified." + except ValueError as error: + assert str(error) == "Image must be specified in the ClusterConfiguration", \ + "Error message did not match expected output." + + def test_cluster_creation_no_mcad(mocker): mocker.patch( "kubernetes.client.CustomObjectsApi.get_cluster_custom_object", From 8f01f0babf2e05d6f3d2ccfbb503e6ae3dec99eb Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Thu, 9 Nov 2023 14:43:24 +0000 Subject: [PATCH 6/6] refactor: update formatting using black --- src/codeflare_sdk/cluster/cluster.py | 5 ++-- tests/unit_test.py | 41 ++++++++++++++++++++++------ tests/unit_test_support.py | 2 +- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/codeflare_sdk/cluster/cluster.py b/src/codeflare_sdk/cluster/cluster.py index a154994d7..59023d8ef 100644 --- a/src/codeflare_sdk/cluster/cluster.py +++ b/src/codeflare_sdk/cluster/cluster.py @@ -131,14 +131,13 @@ def evaluate_dispatch_priority(self): def validate_image_config(self): """ Validates that the image configuration is not empty. - + :param image: The image string to validate :raises ValueError: If the image is not specified """ if self.config.image == "" or self.config.image == None: raise ValueError("Image must be specified in the ClusterConfiguration") - def create_app_wrapper(self): """ Called upon cluster object creation, creates an AppWrapper yaml based on @@ -153,7 +152,7 @@ def create_app_wrapper(self): raise TypeError( f"Namespace {self.config.namespace} is of type {type(self.config.namespace)}. Check your Kubernetes Authentication." ) - + # Validate image configuration self.validate_image_config() diff --git a/tests/unit_test.py b/tests/unit_test.py index 6f9cf8796..14dc8a764 100644 --- a/tests/unit_test.py +++ b/tests/unit_test.py @@ -252,7 +252,6 @@ def test_config_creation(): assert config.dispatch_priority == None assert config.mcad == True assert config.local_interactive == False - def test_cluster_creation(mocker): @@ -262,7 +261,8 @@ def test_cluster_creation(mocker): assert filecmp.cmp( "unit-test-cluster.yaml", f"{parent}/tests/test-case.yaml", shallow=True ) - + + def test_create_app_wrapper_raises_error_with_no_image(): config = createClusterConfig() config.image = "" # Clear the image to test error handling @@ -271,8 +271,9 @@ def test_create_app_wrapper_raises_error_with_no_image(): cluster.create_app_wrapper() assert False, "Expected ValueError when 'image' is not specified." except ValueError as error: - assert str(error) == "Image must be specified in the ClusterConfiguration", \ - "Error message did not match expected output." + assert ( + str(error) == "Image must be specified in the ClusterConfiguration" + ), "Error message did not match expected output." def test_cluster_creation_no_mcad(mocker): @@ -748,7 +749,13 @@ def test_ray_details(mocker, capsys): "codeflare_sdk.cluster.cluster.Cluster.cluster_dashboard_uri", return_value="", ) - cf = Cluster(ClusterConfiguration(name="raytest2", namespace="ns", image= "quay.io/project-codeflare/ray:latest-py39-cu118")) + cf = Cluster( + ClusterConfiguration( + name="raytest2", + namespace="ns", + image="quay.io/project-codeflare/ray:latest-py39-cu118", + ) + ) captured = capsys.readouterr() ray2 = _copy_to_ray(cf) details = cf.details() @@ -1914,7 +1921,13 @@ def test_cluster_status(mocker): head_mem=8, head_gpu=0, ) - cf = Cluster(ClusterConfiguration(name="test", namespace="ns", image="quay.io/project-codeflare/ray:latest-py39-cu118")) + cf = Cluster( + ClusterConfiguration( + name="test", + namespace="ns", + image="quay.io/project-codeflare/ray:latest-py39-cu118", + ) + ) mocker.patch("codeflare_sdk.cluster.cluster._app_wrapper_status", return_value=None) mocker.patch("codeflare_sdk.cluster.cluster._ray_cluster_status", return_value=None) status, ready = cf.status() @@ -2004,7 +2017,13 @@ def test_wait_ready(mocker, capsys): mock_response = mocker.Mock() mock_response.status_code = 200 mocker.patch("requests.get", return_value=mock_response) - cf = Cluster(ClusterConfiguration(name="test", namespace="ns", image= "quay.io/project-codeflare/ray:latest-py39-cu118")) + cf = Cluster( + ClusterConfiguration( + name="test", + namespace="ns", + image="quay.io/project-codeflare/ray:latest-py39-cu118", + ) + ) try: cf.wait_ready(timeout=5) assert 1 == 0 @@ -2669,7 +2688,13 @@ def test_gen_app_wrapper_with_oauth(mocker: MockerFixture): mocker.patch( "codeflare_sdk.utils.generate_yaml.write_user_appwrapper", write_user_appwrapper ) - Cluster(ClusterConfiguration("test_cluster", openshift_oauth=True, image= "quay.io/project-codeflare/ray:latest-py39-cu118")) + Cluster( + ClusterConfiguration( + "test_cluster", + openshift_oauth=True, + image="quay.io/project-codeflare/ray:latest-py39-cu118", + ) + ) user_yaml = write_user_appwrapper.call_args.args[0] assert any( container["name"] == "oauth-proxy" diff --git a/tests/unit_test_support.py b/tests/unit_test_support.py index 4c11c5930..85f1a76d3 100644 --- a/tests/unit_test_support.py +++ b/tests/unit_test_support.py @@ -47,7 +47,7 @@ def createClusterConfig(): machine_types=["cpu.small", "gpu.large"], image_pull_secrets=["unit-test-pull-secret"], ingress_domain="apps.cluster.awsroute.org", - image="quay.io/project-codeflare/ray:latest-py39-cu118" + image="quay.io/project-codeflare/ray:latest-py39-cu118", ) return config