Skip to content

Commit d04b6ac

Browse files
Bobbins228Bobbins228
committed
Ray mTLS for basic Interactive
1 parent 8132578 commit d04b6ac

File tree

4 files changed

+190
-36
lines changed

4 files changed

+190
-36
lines changed

Diff for: ‎demo-notebooks/additional-demos/local_interactive.ipynb

+138-29
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"cells": [
33
{
44
"cell_type": "code",
5-
"execution_count": null,
5+
"execution_count": 1,
66
"id": "9a44568b-61ef-41c7-8ad1-9a3b128f03a7",
77
"metadata": {
88
"tags": []
@@ -36,9 +36,7 @@
3636
"source": [
3737
"\n",
3838
"NOTE: We must specify the `image` which will be used in our RayCluster, we recommend you bring your own image which suits your purposes. \n",
39-
"The example here is a community image.\n",
40-
"\n",
41-
"NOTE: By default the SDK uses Kueue as it's scheduling solution to use MCAD set the `mcad=True` option in `ClusterConfiguration`"
39+
"The example here is a community image."
4240
]
4341
},
4442
{
@@ -50,29 +48,24 @@
5048
},
5149
"outputs": [],
5250
"source": [
53-
"# Create our cluster and submit\n",
54-
"# The SDK will try to find the name of your default local queue based on the annotation \"kueue.x-k8s.io/default-queue\": \"true\"\n",
51+
"# Create and submit our Ray Cluster\n",
5552
"namespace = \"default\"\n",
5653
"cluster_name = \"hfgputest-1\"\n",
57-
"local_interactive = True\n",
5854
"\n",
59-
"cluster = Cluster(ClusterConfiguration(local_interactive=local_interactive,\n",
60-
" namespace=namespace,\n",
55+
"cluster = Cluster(ClusterConfiguration(namespace=namespace,\n",
6156
" name=cluster_name,\n",
6257
" num_workers=1,\n",
6358
" min_cpus=1,\n",
6459
" max_cpus=1,\n",
6560
" min_memory=4,\n",
6661
" max_memory=4,\n",
6762
" num_gpus=0,\n",
68-
" image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n",
69-
" # local_queue=\"local-queue-name\" # Specify the local queue manually\n",
70-
" ))"
63+
" image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",))"
7164
]
7265
},
7366
{
7467
"cell_type": "code",
75-
"execution_count": null,
68+
"execution_count": 3,
7669
"id": "69968140-15e6-482f-9529-82b0cd19524b",
7770
"metadata": {
7871
"tags": []
@@ -84,12 +77,21 @@
8477
},
8578
{
8679
"cell_type": "code",
87-
"execution_count": null,
80+
"execution_count": 4,
8881
"id": "e20f9982-f671-460b-8c22-3d62e101fed9",
8982
"metadata": {
9083
"tags": []
9184
},
92-
"outputs": [],
85+
"outputs": [
86+
{
87+
"name": "stdout",
88+
"output_type": "stream",
89+
"text": [
90+
"Waiting for requested resources to be set up...\n",
91+
"Requested cluster up and running!\n"
92+
]
93+
}
94+
],
9395
"source": [
9496
"cluster.wait_ready()"
9597
]
@@ -110,21 +112,89 @@
110112
"outputs": [],
111113
"source": [
112114
"from codeflare_sdk import generate_cert\n",
113-
"\n",
114-
"if local_interactive:\n",
115-
" generate_cert.generate_tls_cert(cluster_name, namespace)\n",
116-
" generate_cert.export_env(cluster_name, namespace)"
115+
"# Export the environment variables to enable TLS\n",
116+
"generate_cert.generate_tls_cert(cluster_name, namespace)\n",
117+
"generate_cert.export_env(cluster_name, namespace)"
117118
]
118119
},
119120
{
120121
"cell_type": "code",
121-
"execution_count": null,
122+
"execution_count": 6,
122123
"id": "9483bb98-33b3-4beb-9b15-163d7e76c1d7",
123124
"metadata": {
124125
"scrolled": true,
125126
"tags": []
126127
},
127-
"outputs": [],
128+
"outputs": [
129+
{
130+
"name": "stderr",
131+
"output_type": "stream",
132+
"text": [
133+
"2023-06-27 19:14:16,088\tINFO client_builder.py:251 -- Passing the following kwargs to ray.init() on the server: logging_level\n",
134+
"2023-06-27 19:14:16,100\tDEBUG worker.py:378 -- client gRPC channel state change: ChannelConnectivity.IDLE\n",
135+
"2023-06-27 19:14:16,308\tDEBUG worker.py:378 -- client gRPC channel state change: ChannelConnectivity.CONNECTING\n",
136+
"2023-06-27 19:14:16,434\tDEBUG worker.py:378 -- client gRPC channel state change: ChannelConnectivity.READY\n",
137+
"2023-06-27 19:14:16,436\tDEBUG worker.py:807 -- Pinging server.\n",
138+
"2023-06-27 19:14:18,634\tDEBUG worker.py:640 -- Retaining 00ffffffffffffffffffffffffffffffffffffff0100000001000000\n",
139+
"2023-06-27 19:14:18,635\tDEBUG worker.py:564 -- Scheduling task get_dashboard_url 0 b'\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00'\n",
140+
"2023-06-27 19:14:18,645\tDEBUG worker.py:640 -- Retaining c8ef45ccd0112571ffffffffffffffffffffffff0100000001000000\n",
141+
"2023-06-27 19:14:19,454\tDEBUG worker.py:636 -- Releasing c8ef45ccd0112571ffffffffffffffffffffffff0100000001000000\n"
142+
]
143+
},
144+
{
145+
"data": {
146+
"text/html": [
147+
"<div>\n",
148+
" <div style=\"margin-left: 50px;display: flex;flex-direction: row;align-items: center\">\n",
149+
" <h3 style=\"color: var(--jp-ui-font-color0)\">Ray</h3>\n",
150+
" <svg version=\"1.1\" id=\"ray\" width=\"3em\" viewBox=\"0 0 144.5 144.6\" style=\"margin-left: 3em;margin-right: 3em\">\n",
151+
" <g id=\"layer-1\">\n",
152+
" <path fill=\"#00a2e9\" class=\"st0\" d=\"M97.3,77.2c-3.8-1.1-6.2,0.9-8.3,5.1c-3.5,6.8-9.9,9.9-17.4,9.6S58,88.1,54.8,81.2c-1.4-3-3-4-6.3-4.1\n",
153+
" c-5.6-0.1-9.9,0.1-13.1,6.4c-3.8,7.6-13.6,10.2-21.8,7.6C5.2,88.4-0.4,80.5,0,71.7c0.1-8.4,5.7-15.8,13.8-18.2\n",
154+
" c8.4-2.6,17.5,0.7,22.3,8c1.3,1.9,1.3,5.2,3.6,5.6c3.9,0.6,8,0.2,12,0.2c1.8,0,1.9-1.6,2.4-2.8c3.5-7.8,9.7-11.8,18-11.9\n",
155+
" c8.2-0.1,14.4,3.9,17.8,11.4c1.3,2.8,2.9,3.6,5.7,3.3c1-0.1,2,0.1,3,0c2.8-0.5,6.4,1.7,8.1-2.7s-2.3-5.5-4.1-7.5\n",
156+
" c-5.1-5.7-10.9-10.8-16.1-16.3C84,38,81.9,37.1,78,38.3C66.7,42,56.2,35.7,53,24.1C50.3,14,57.3,2.8,67.7,0.5\n",
157+
" C78.4-2,89,4.7,91.5,15.3c0.1,0.3,0.1,0.5,0.2,0.8c0.7,3.4,0.7,6.9-0.8,9.8c-1.7,3.2-0.8,5,1.5,7.2c6.7,6.5,13.3,13,19.8,19.7\n",
158+
" c1.8,1.8,3,2.1,5.5,1.2c9.1-3.4,17.9-0.6,23.4,7c4.8,6.9,4.6,16.1-0.4,22.9c-5.4,7.2-14.2,9.9-23.1,6.5c-2.3-0.9-3.5-0.6-5.1,1.1\n",
159+
" c-6.7,6.9-13.6,13.7-20.5,20.4c-1.8,1.8-2.5,3.2-1.4,5.9c3.5,8.7,0.3,18.6-7.7,23.6c-7.9,5-18.2,3.8-24.8-2.9\n",
160+
" c-6.4-6.4-7.4-16.2-2.5-24.3c4.9-7.8,14.5-11,23.1-7.8c3,1.1,4.7,0.5,6.9-1.7C91.7,98.4,98,92.3,104.2,86c1.6-1.6,4.1-2.7,2.6-6.2\n",
161+
" c-1.4-3.3-3.8-2.5-6.2-2.6C99.8,77.2,98.9,77.2,97.3,77.2z M72.1,29.7c5.5,0.1,9.9-4.3,10-9.8c0-0.1,0-0.2,0-0.3\n",
162+
" C81.8,14,77,9.8,71.5,10.2c-5,0.3-9,4.2-9.3,9.2c-0.2,5.5,4,10.1,9.5,10.3C71.8,29.7,72,29.7,72.1,29.7z M72.3,62.3\n",
163+
" c-5.4-0.1-9.9,4.2-10.1,9.7c0,0.2,0,0.3,0,0.5c0.2,5.4,4.5,9.7,9.9,10c5.1,0.1,9.9-4.7,10.1-9.8c0.2-5.5-4-10-9.5-10.3\n",
164+
" C72.6,62.3,72.4,62.3,72.3,62.3z M115,72.5c0.1,5.4,4.5,9.7,9.8,9.9c5.6-0.2,10-4.8,10-10.4c-0.2-5.4-4.6-9.7-10-9.7\n",
165+
" c-5.3-0.1-9.8,4.2-9.9,9.5C115,72.1,115,72.3,115,72.5z M19.5,62.3c-5.4,0.1-9.8,4.4-10,9.8c-0.1,5.1,5.2,10.4,10.2,10.3\n",
166+
" c5.6-0.2,10-4.9,9.8-10.5c-0.1-5.4-4.5-9.7-9.9-9.6C19.6,62.3,19.5,62.3,19.5,62.3z M71.8,134.6c5.9,0.2,10.3-3.9,10.4-9.6\n",
167+
" c0.5-5.5-3.6-10.4-9.1-10.8c-5.5-0.5-10.4,3.6-10.8,9.1c0,0.5,0,0.9,0,1.4c-0.2,5.3,4,9.8,9.3,10\n",
168+
" C71.6,134.6,71.7,134.6,71.8,134.6z\"/>\n",
169+
" </g>\n",
170+
" </svg>\n",
171+
" <table>\n",
172+
" <tr>\n",
173+
" <td style=\"text-align: left\"><b>Python version:</b></td>\n",
174+
" <td style=\"text-align: left\"><b>3.8.13</b></td>\n",
175+
" </tr>\n",
176+
" <tr>\n",
177+
" <td style=\"text-align: left\"><b>Ray version:</b></td>\n",
178+
" <td style=\"text-align: left\"><b> 2.1.0</b></td>\n",
179+
" </tr>\n",
180+
" <tr>\n",
181+
" <td style=\"text-align: left\"><b>Dashboard:</b></td>\n",
182+
" <td style=\"text-align: left\"><b><a href=\"http://10.254.20.41:8265\" target=\"_blank\">http://10.254.20.41:8265</a></b></td>\n",
183+
"</tr>\n",
184+
"\n",
185+
" </table>\n",
186+
" </div>\n",
187+
"</div>\n"
188+
],
189+
"text/plain": [
190+
"ClientContext(dashboard_url='10.254.20.41:8265', python_version='3.8.13', ray_version='2.1.0', ray_commit='23f34d948dae8de9b168667ab27e6cf940b3ae85', protocol_version='2022-10-05', _num_clients=1, _context_to_restore=<ray.util.client._ClientContext object at 0x108ca2730>)"
191+
]
192+
},
193+
"execution_count": 6,
194+
"metadata": {},
195+
"output_type": "execute_result"
196+
}
197+
],
128198
"source": [
129199
"import ray\n",
130200
"\n",
@@ -134,7 +204,7 @@
134204
},
135205
{
136206
"cell_type": "code",
137-
"execution_count": null,
207+
"execution_count": 7,
138208
"id": "3436eb4a-217c-4109-a3c3-309fda7e2442",
139209
"metadata": {},
140210
"outputs": [],
@@ -158,41 +228,80 @@
158228
},
159229
{
160230
"cell_type": "code",
161-
"execution_count": null,
231+
"execution_count": 8,
162232
"id": "5cca1874-2be3-4631-ae48-9adfa45e3af3",
163233
"metadata": {
164234
"scrolled": true,
165235
"tags": []
166236
},
167-
"outputs": [],
237+
"outputs": [
238+
{
239+
"name": "stderr",
240+
"output_type": "stream",
241+
"text": [
242+
"2023-06-27 19:14:28,222\tDEBUG worker.py:640 -- Retaining 00ffffffffffffffffffffffffffffffffffffff0100000002000000\n",
243+
"2023-06-27 19:14:28,222\tDEBUG worker.py:564 -- Scheduling task heavy_calculation 0 b'\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x01\\x00\\x00\\x00\\x02\\x00\\x00\\x00'\n"
244+
]
245+
}
246+
],
168247
"source": [
169248
"ref = heavy_calculation.remote(3000)"
170249
]
171250
},
172251
{
173252
"cell_type": "code",
174-
"execution_count": null,
253+
"execution_count": 9,
175254
"id": "01172c29-e8bf-41ef-8db5-eccb07906111",
176255
"metadata": {},
177-
"outputs": [],
256+
"outputs": [
257+
{
258+
"name": "stderr",
259+
"output_type": "stream",
260+
"text": [
261+
"2023-06-27 19:14:29,202\tDEBUG worker.py:640 -- Retaining 16310a0f0a45af5cffffffffffffffffffffffff0100000001000000\n",
262+
"2023-06-27 19:14:31,224\tDEBUG worker.py:439 -- Internal retry for get [ClientObjectRef(16310a0f0a45af5cffffffffffffffffffffffff0100000001000000)]\n"
263+
]
264+
},
265+
{
266+
"data": {
267+
"text/plain": [
268+
"1789.4644387076714"
269+
]
270+
},
271+
"execution_count": 9,
272+
"metadata": {},
273+
"output_type": "execute_result"
274+
}
275+
],
178276
"source": [
179277
"ray.get(ref)"
180278
]
181279
},
182280
{
183281
"cell_type": "code",
184-
"execution_count": null,
282+
"execution_count": 10,
185283
"id": "9e79b547-a457-4232-b77d-19147067b972",
186284
"metadata": {},
187-
"outputs": [],
285+
"outputs": [
286+
{
287+
"name": "stderr",
288+
"output_type": "stream",
289+
"text": [
290+
"2023-06-27 19:14:33,161\tDEBUG dataclient.py:287 -- Got unawaited response connection_cleanup {\n",
291+
"}\n",
292+
"\n",
293+
"2023-06-27 19:14:34,460\tDEBUG dataclient.py:278 -- Shutting down data channel.\n"
294+
]
295+
}
296+
],
188297
"source": [
189298
"ray.cancel(ref)\n",
190299
"ray.shutdown()"
191300
]
192301
},
193302
{
194303
"cell_type": "code",
195-
"execution_count": null,
304+
"execution_count": 11,
196305
"id": "2c198f1f-68bf-43ff-a148-02b5cb000ff2",
197306
"metadata": {},
198307
"outputs": [],

Diff for: ‎demo-notebooks/guided-demos/3_basic_interactive.ipynb

+17-2
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,11 @@
5959
"source": [
6060
"# Create and configure our cluster object\n",
6161
"# The SDK will try to find the name of your default local queue based on the annotation \"kueue.x-k8s.io/default-queue\": \"true\"\n",
62+
"namespace = \"default\"\n",
63+
"cluster_name = \"interactivetest\"\n",
6264
"cluster = Cluster(ClusterConfiguration(\n",
63-
" name='interactivetest',\n",
64-
" namespace='default',\n",
65+
" name=cluster_name,\n",
66+
" namespace=namespace,\n",
6567
" num_workers=2,\n",
6668
" min_cpus=2,\n",
6769
" max_cpus=2,\n",
@@ -126,6 +128,19 @@
126128
"Now we can connect directly to our Ray cluster via the Ray python client:"
127129
]
128130
},
131+
{
132+
"cell_type": "code",
133+
"execution_count": null,
134+
"id": "c9436436",
135+
"metadata": {},
136+
"outputs": [],
137+
"source": [
138+
"from codeflare_sdk import generate_cert\n",
139+
"# Export the environment variables to enable TLS\n",
140+
"generate_cert.generate_tls_cert(cluster_name, namespace)\n",
141+
"generate_cert.export_env(cluster_name, namespace)"
142+
]
143+
},
129144
{
130145
"cell_type": "code",
131146
"execution_count": null,

Diff for: ‎demo-notebooks/guided-demos/notebook-ex-outputs/3_basic_interactive.ipynb

+17-2
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,11 @@
6767
"source": [
6868
"# Create and configure our cluster object\n",
6969
"# The SDK will try to find the name of your default local queue based on the annotation \"kueue.x-k8s.io/default-queue\": \"true\"\n",
70+
"namespace = \"default\"\n",
71+
"cluster_name = \"interactivetest\"\n",
7072
"cluster = Cluster(ClusterConfiguration(\n",
71-
" name='interactivetest',\n",
72-
" namespace='default',\n",
73+
" name=cluster_name,\n",
74+
" namespace=namespace,\n",
7375
" num_workers=2,\n",
7476
" min_cpus=2,\n",
7577
" max_cpus=2,\n",
@@ -210,6 +212,19 @@
210212
"Now we can connect directly to our Ray cluster via the Ray python client:"
211213
]
212214
},
215+
{
216+
"cell_type": "code",
217+
"execution_count": null,
218+
"id": "13eb52f6",
219+
"metadata": {},
220+
"outputs": [],
221+
"source": [
222+
"from codeflare_sdk import generate_cert\n",
223+
"# Export the environment variables to enable TLS\n",
224+
"generate_cert.generate_tls_cert(cluster_name, namespace)\n",
225+
"generate_cert.export_env(cluster_name, namespace)"
226+
]
227+
},
213228
{
214229
"cell_type": "code",
215230
"execution_count": 6,

Diff for: ‎demo-notebooks/guided-demos/preview_nbs/3_basic_interactive.ipynb

+18-3
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,11 @@
5959
"source": [
6060
"# Create and configure our cluster object\n",
6161
"# The SDK will try to find the name of your default local queue based on the annotation \"kueue.x-k8s.io/default-queue\": \"true\"\n",
62+
"namespace = \"default\"\n",
63+
"cluster_name = \"interactivetest\"\n",
6264
"cluster = Cluster(ClusterConfiguration(\n",
63-
" name='interactivetest',\n",
64-
" namespace='default',\n",
65+
" name=cluster_name,\n",
66+
" namespace=namespace,\n",
6567
" num_workers=2,\n",
6668
" min_cpus=2,\n",
6769
" max_cpus=2,\n",
@@ -126,6 +128,19 @@
126128
"Now we can connect directly to our Ray cluster via the Ray python client:"
127129
]
128130
},
131+
{
132+
"cell_type": "code",
133+
"execution_count": null,
134+
"id": "e5308271",
135+
"metadata": {},
136+
"outputs": [],
137+
"source": [
138+
"from codeflare_sdk import generate_cert\n",
139+
"# Export the environment variables to enable TLS\n",
140+
"generate_cert.generate_tls_cert(cluster_name, namespace)\n",
141+
"generate_cert.export_env(cluster_name, namespace)"
142+
]
143+
},
129144
{
130145
"cell_type": "code",
131146
"execution_count": null,
@@ -296,7 +311,7 @@
296311
"name": "python",
297312
"nbconvert_exporter": "python",
298313
"pygments_lexer": "ipython3",
299-
"version": "3.8.13"
314+
"version": "3.9.18"
300315
},
301316
"vscode": {
302317
"interpreter": {

0 commit comments

Comments
 (0)