diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index dbf2fce24..bd4e1024f 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -64,11 +64,12 @@ jobs: with: repository: project-codeflare/codeflare-operator path: codeflare-operator + ref: mcadv1b2 - name: Set Go uses: actions/setup-go@v5 with: - go-version: v1.20 + go-version-file: './go.mod' - name: Set up gotestfmt uses: gotesttools/gotestfmt-action@v2 @@ -78,10 +79,15 @@ jobs: - name: Setup and start KinD cluster uses: ./common/github-actions/kind + - name: Deploy Kueue + working-directory: codeflare-operator + run: | + make kueue-e2e + - name: Deploy CodeFlare stack id: deploy + working-directory: codeflare-operator run: | - cd codeflare-operator echo Deploying CodeFlare operator IMG="${REGISTRY_ADDRESS}"/codeflare-operator make image-push -e IMG="${IMG}" @@ -91,8 +97,6 @@ jobs: echo Setting up CodeFlare stack make setup-e2e - cd .. - - name: Add user to KinD uses: ./common/github-actions/kind-add-user with: @@ -122,6 +126,7 @@ jobs: poetry run pytest -v -s ./tests/e2e -m kind > ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log 2>&1 - name: Switch to kind-cluster context to print logs + if: always() && steps.deploy.outcome == 'success' run: kubectl config use-context kind-cluster - name: Print CodeFlare operator logs @@ -130,6 +135,13 @@ jobs: echo "Printing CodeFlare operator logs" kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log + - name: Print Kueue operator logs + if: always() && steps.deploy.outcome == 'success' + run: | + echo "Printing Kueue operator logs" + KUEUE_CONTROLLER_POD=$(kubectl get pods -n kueue-system | grep kueue-controller | awk '{print $1}') + kubectl logs -n kueue-system --tail -1 ${KUEUE_CONTROLLER_POD} | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kueue.log + - name: Print KubeRay operator logs if: always() && steps.deploy.outcome == 'success' run: | diff --git a/demo-notebooks/guided-demos/0_basic_ray.ipynb b/demo-notebooks/guided-demos/0_basic_ray.ipynb index 205f02175..9e7b8a39e 100644 --- a/demo-notebooks/guided-demos/0_basic_ray.ipynb +++ b/demo-notebooks/guided-demos/0_basic_ray.ipynb @@ -69,7 +69,6 @@ " max_memory=4,\n", " num_gpus=0,\n", " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n", - " instascale=False\n", "))" ] }, diff --git a/demo-notebooks/guided-demos/1_basic_instascale.ipynb b/demo-notebooks/guided-demos/1_basic_instascale.ipynb deleted file mode 100644 index 418737eb6..000000000 --- a/demo-notebooks/guided-demos/1_basic_instascale.ipynb +++ /dev/null @@ -1,177 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "9865ee8c", - "metadata": {}, - "source": [ - "In this second notebook, we will go over the basics of using InstaScale to scale up/down necessary resources that are not currently available on your OpenShift Cluster (in cloud environments)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b55bc3ea-4ce3-49bf-bb1f-e209de8ca47a", - "metadata": {}, - "outputs": [], - "source": [ - "# Import pieces from codeflare-sdk\n", - "from codeflare_sdk import Cluster, ClusterConfiguration, TokenAuthentication" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "614daa0c", - "metadata": {}, - "outputs": [], - "source": [ - "# Create authentication object for user permissions\n", - "# IF unused, SDK will automatically check for default kubeconfig, then in-cluster config\n", - "# KubeConfigFileAuthentication can also be used to specify kubeconfig path manually\n", - "auth = TokenAuthentication(\n", - " token = \"XXXXX\",\n", - " server = \"XXXXX\",\n", - " skip_tls=False\n", - ")\n", - "auth.login()" - ] - }, - { - "cell_type": "markdown", - "id": "bc27f84c", - "metadata": {}, - "source": [ - "This time, we are working in a cloud environment, and our OpenShift cluster does not have the resources needed for our desired workloads. We will use InstaScale to dynamically scale-up guaranteed resources based on our request (that will also automatically scale-down when we are finished working):\n", - "\n", - "NOTE: We must specify the `image` which will be used in our RayCluster, we recommend you bring your own image which suits your purposes. \n", - "The example here is a community image." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0f4bc870-091f-4e11-9642-cba145710159", - "metadata": {}, - "outputs": [], - "source": [ - "# Create and configure our cluster object (and appwrapper)\n", - "cluster = Cluster(ClusterConfiguration(\n", - " name='instascaletest',\n", - " namespace='default',\n", - " num_workers=2,\n", - " min_cpus=2,\n", - " max_cpus=2,\n", - " min_memory=8,\n", - " max_memory=8,\n", - " num_gpus=1,\n", - " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n", - " instascale=True, # InstaScale now enabled, will scale OCP cluster to guarantee resource request\n", - " machine_types=[\"m5.xlarge\", \"g4dn.xlarge\"] # Head, worker AWS machine types desired\n", - "))" - ] - }, - { - "cell_type": "markdown", - "id": "12eef53c", - "metadata": {}, - "source": [ - "Same as last time, we will bring the cluster up, wait for it to be ready, and confirm that the specs are as-requested:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f0884bbc-c224-4ca0-98a0-02dfa09c2200", - "metadata": {}, - "outputs": [], - "source": [ - "# Bring up the cluster\n", - "cluster.up()\n", - "cluster.wait_ready()" - ] - }, - { - "cell_type": "markdown", - "id": "6abfe904", - "metadata": {}, - "source": [ - "While the resources are being scaled, we can also go into the console and take a look at the InstaScale logs, as well as the new machines/nodes spinning up.\n", - "\n", - "Once the cluster is ready, we can confirm the specs:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7fd45bc5-03c0-4ae5-9ec5-dd1c30f1a084", - "metadata": {}, - "outputs": [], - "source": [ - "cluster.details()" - ] - }, - { - "cell_type": "markdown", - "id": "5af8cd32", - "metadata": {}, - "source": [ - "Finally, we bring our resource cluster down and release/terminate the associated resources, bringing everything back to the way it was before our cluster was brought up." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5f36db0f-31f6-4373-9503-dc3c1c4c3f57", - "metadata": {}, - "outputs": [], - "source": [ - "cluster.down()" - ] - }, - { - "cell_type": "markdown", - "id": "c883caea", - "metadata": {}, - "source": [ - "Once again, we can look at the machines/nodes and see that everything has been successfully scaled down!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0d41b90e", - "metadata": {}, - "outputs": [], - "source": [ - "auth.logout()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.13" - }, - "vscode": { - "interpreter": { - "hash": "f9f85f796d01129d0dd105a088854619f454435301f6ffec2fea96ecbd9be4ac" - } - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/demo-notebooks/guided-demos/3_basic_interactive.ipynb b/demo-notebooks/guided-demos/3_basic_interactive.ipynb index 090a4a305..93c03d80a 100644 --- a/demo-notebooks/guided-demos/3_basic_interactive.ipynb +++ b/demo-notebooks/guided-demos/3_basic_interactive.ipynb @@ -65,10 +65,7 @@ " min_memory=8,\n", " max_memory=8,\n", " num_gpus=1,\n", - " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n", - " instascale=True, #<---instascale enabled\n", - " machine_types=[\"m5.xlarge\", \"g4dn.xlarge\"]\n", - " \n", + " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\", \n", "))" ] }, diff --git a/demo-notebooks/guided-demos/4_gpt.ipynb b/demo-notebooks/guided-demos/4_gpt.ipynb index a4e1f366e..49f73f64c 100644 --- a/demo-notebooks/guided-demos/4_gpt.ipynb +++ b/demo-notebooks/guided-demos/4_gpt.ipynb @@ -55,8 +55,6 @@ " max_memory=8,\n", " num_gpus=1,\n", " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n", - " instascale=True, #<---instascale enabled\n", - " machine_types=[\"m5.xlarge\", \"g4dn.xlarge\"],\n", "))" ] }, diff --git a/demo-notebooks/guided-demos/notebook-ex-outputs/1_basic_instascale.ipynb b/demo-notebooks/guided-demos/notebook-ex-outputs/1_basic_instascale.ipynb deleted file mode 100644 index 4b28b2058..000000000 --- a/demo-notebooks/guided-demos/notebook-ex-outputs/1_basic_instascale.ipynb +++ /dev/null @@ -1,252 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "9865ee8c", - "metadata": {}, - "source": [ - "In this second notebook, we will go over the basics of using InstaScale to scale up/down necessary resources that are not currently available on your OpenShift Cluster (in cloud environments)." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "b55bc3ea-4ce3-49bf-bb1f-e209de8ca47a", - "metadata": {}, - "outputs": [], - "source": [ - "# Import pieces from codeflare-sdk\n", - "from codeflare_sdk import Cluster, ClusterConfiguration, TokenAuthentication" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "614daa0c", - "metadata": {}, - "outputs": [], - "source": [ - "# Create authentication object for user permissions\n", - "# IF unused, SDK will automatically check for default kubeconfig, then in-cluster config\n", - "# KubeConfigFileAuthentication can also be used to specify kubeconfig path manually\n", - "auth = TokenAuthentication(\n", - " token = \"XXXXX\",\n", - " server = \"XXXXX\",\n", - " skip_tls=False\n", - ")\n", - "auth.login()" - ] - }, - { - "cell_type": "markdown", - "id": "bc27f84c", - "metadata": {}, - "source": [ - "This time, we are working in a cloud environment, and our OpenShift cluster does not have the resources needed for our desired workloads. We will use InstaScale to dynamically scale-up guaranteed resources based on our request (that will also automatically scale-down when we are finished working):\n", - "\n", - "NOTE: We must specify the `image` which will be used in our RayCluster, we recommend you bring your own image which suits your purposes. \n", - "The example here is a community image." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "0f4bc870-091f-4e11-9642-cba145710159", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Written to: instascaletest.yaml\n" - ] - } - ], - "source": [ - "# Create and configure our cluster object (and appwrapper)\n", - "cluster = Cluster(ClusterConfiguration(\n", - " name='instascaletest',\n", - " namespace='default',\n", - " num_workers=2,\n", - " min_cpus=2,\n", - " max_cpus=2,\n", - " min_memory=8,\n", - " max_memory=8,\n", - " num_gpus=1,\n", - " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n", - " instascale=True, # InstaScale now enabled, will scale OCP cluster to guarantee resource request\n", - " machine_types=[\"m5.xlarge\", \"g4dn.xlarge\"] # Head, worker AWS machine types desired\n", - "))" - ] - }, - { - "cell_type": "markdown", - "id": "12eef53c", - "metadata": {}, - "source": [ - "Same as last time, we will bring the cluster up, wait for it to be ready, and confirm that the specs are as-requested:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "f0884bbc-c224-4ca0-98a0-02dfa09c2200", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Waiting for requested resources to be set up...\n", - "Requested cluster up and running!\n" - ] - } - ], - "source": [ - "# Bring up the cluster\n", - "cluster.up()\n", - "cluster.wait_ready()" - ] - }, - { - "cell_type": "markdown", - "id": "6abfe904", - "metadata": {}, - "source": [ - "While the resources are being scaled, we can also go into the console and take a look at the InstaScale logs, as well as the new machines/nodes spinning up.\n", - "\n", - "Once the cluster is ready, we can confirm the specs:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "7fd45bc5-03c0-4ae5-9ec5-dd1c30f1a084", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
                     ๐Ÿš€ CodeFlare Cluster Details ๐Ÿš€                     \n",
-       "                                                                         \n",
-       " โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ \n",
-       " โ”‚   Name                                                              โ”‚ \n",
-       " โ”‚   instascaletest                                        Active โœ…   โ”‚ \n",
-       " โ”‚                                                                     โ”‚ \n",
-       " โ”‚   URI: ray://instascaletest-head-svc.default.svc:10001              โ”‚ \n",
-       " โ”‚                                                                     โ”‚ \n",
-       " โ”‚   Dashboard๐Ÿ”—                                                       โ”‚ \n",
-       " โ”‚                                                                     โ”‚ \n",
-       " โ”‚                       Cluster Resources                             โ”‚ \n",
-       " โ”‚   โ•ญโ”€โ”€ Workers โ”€โ”€โ•ฎ  โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ Worker specs(each) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ         โ”‚ \n",
-       " โ”‚   โ”‚  # Workers  โ”‚  โ”‚  Memory      CPU         GPU         โ”‚         โ”‚ \n",
-       " โ”‚   โ”‚             โ”‚  โ”‚                                      โ”‚         โ”‚ \n",
-       " โ”‚   โ”‚  2          โ”‚  โ”‚  8~8         2           1           โ”‚         โ”‚ \n",
-       " โ”‚   โ”‚             โ”‚  โ”‚                                      โ”‚         โ”‚ \n",
-       " โ”‚   โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ  โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ         โ”‚ \n",
-       " โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ \n",
-       "
\n" - ], - "text/plain": [ - "\u001b[3m \u001b[0m\u001b[1;3m ๐Ÿš€ CodeFlare Cluster Details ๐Ÿš€\u001b[0m\u001b[3m \u001b[0m\n", - "\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\n", - " โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ \n", - " โ”‚ \u001b[1;37;42mName\u001b[0m โ”‚ \n", - " โ”‚ \u001b[1;4minstascaletest\u001b[0m Active โœ… โ”‚ \n", - " โ”‚ โ”‚ \n", - " โ”‚ \u001b[1mURI:\u001b[0m ray://instascaletest-head-svc.default.svc:10001 โ”‚ \n", - " โ”‚ โ”‚ \n", - " โ”‚ \u001b]8;id=65933;http://ray-dashboard-instascaletest-default.apps.meyceoz-07122023.psap.aws.rhperfscale.org\u001b\\\u001b[4;34mDashboard๐Ÿ”—\u001b[0m\u001b]8;;\u001b\\ โ”‚ \n", - " โ”‚ โ”‚ \n", - " โ”‚ \u001b[3m Cluster Resources \u001b[0m โ”‚ \n", - " โ”‚ โ•ญโ”€โ”€ Workers โ”€โ”€โ•ฎ โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ Worker specs(each) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ โ”‚ \n", - " โ”‚ โ”‚ \u001b[1m \u001b[0m\u001b[1m# Workers\u001b[0m\u001b[1m \u001b[0m โ”‚ โ”‚ \u001b[1m \u001b[0m\u001b[1mMemory \u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1mCPU \u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1mGPU \u001b[0m\u001b[1m \u001b[0m โ”‚ โ”‚ \n", - " โ”‚ โ”‚ \u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m โ”‚ โ”‚ \u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m โ”‚ โ”‚ \n", - " โ”‚ โ”‚ \u001b[35m \u001b[0m\u001b[35m2 \u001b[0m\u001b[35m \u001b[0m โ”‚ โ”‚ \u001b[36m \u001b[0m\u001b[36m8~8 \u001b[0m\u001b[36m \u001b[0m\u001b[35m \u001b[0m\u001b[35m2 \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m1 \u001b[0m\u001b[35m \u001b[0m โ”‚ โ”‚ \n", - " โ”‚ โ”‚ \u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m โ”‚ โ”‚ \u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m โ”‚ โ”‚ \n", - " โ”‚ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ”‚ \n", - " โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ \n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [ - "RayCluster(name='instascaletest', status=, workers=2, worker_mem_min=8, worker_mem_max=8, worker_cpu=2, worker_gpu=1, namespace='default', dashboard='http://ray-dashboard-instascaletest-default.apps.meyceoz-07122023.psap.aws.rhperfscale.org')" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cluster.details()" - ] - }, - { - "cell_type": "markdown", - "id": "5af8cd32", - "metadata": {}, - "source": [ - "Finally, we bring our resource cluster down and release/terminate the associated resources, bringing everything back to the way it was before our cluster was brought up." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "5f36db0f-31f6-4373-9503-dc3c1c4c3f57", - "metadata": {}, - "outputs": [], - "source": [ - "cluster.down()" - ] - }, - { - "cell_type": "markdown", - "id": "c883caea", - "metadata": {}, - "source": [ - "Once again, we can look at the machines/nodes and see that everything has been successfully scaled down!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0d41b90e", - "metadata": {}, - "outputs": [], - "source": [ - "auth.logout()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.17" - }, - "vscode": { - "interpreter": { - "hash": "f9f85f796d01129d0dd105a088854619f454435301f6ffec2fea96ecbd9be4ac" - } - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/demo-notebooks/guided-demos/notebook-ex-outputs/instascaletest.yaml b/demo-notebooks/guided-demos/notebook-ex-outputs/instascaletest.yaml deleted file mode 100644 index 8cb96a794..000000000 --- a/demo-notebooks/guided-demos/notebook-ex-outputs/instascaletest.yaml +++ /dev/null @@ -1,185 +0,0 @@ -apiVersion: workload.codeflare.dev/v1beta1 -kind: AppWrapper -metadata: - labels: - orderedinstance: m5.xlarge_g4dn.xlarge - name: instascaletest - namespace: default -spec: - priority: 9 - resources: - GenericItems: - - custompodresources: - - limits: - cpu: 2 - memory: 8G - nvidia.com/gpu: 0 - replicas: 1 - requests: - cpu: 2 - memory: 8G - nvidia.com/gpu: 0 - - limits: - cpu: 2 - memory: 8G - nvidia.com/gpu: 1 - replicas: 2 - requests: - cpu: 2 - memory: 8G - nvidia.com/gpu: 1 - generictemplate: - apiVersion: ray.io/v1 - kind: RayCluster - metadata: - labels: - appwrapper.mcad.ibm.com: instascaletest - controller-tools.k8s.io: '1.0' - name: instascaletest - namespace: default - spec: - autoscalerOptions: - idleTimeoutSeconds: 60 - imagePullPolicy: Always - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 500m - memory: 512Mi - upscalingMode: Default - enableInTreeAutoscaling: false - headGroupSpec: - rayStartParams: - block: 'true' - dashboard-host: 0.0.0.0 - num-gpus: '0' - serviceType: ClusterIP - template: - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: instascaletest - operator: In - values: - - instascaletest - containers: - - env: - - name: MY_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: RAY_USE_TLS - value: '0' - - name: RAY_TLS_SERVER_CERT - value: /home/ray/workspace/tls/server.crt - - name: RAY_TLS_SERVER_KEY - value: /home/ray/workspace/tls/server.key - - name: RAY_TLS_CA_CERT - value: /home/ray/workspace/tls/ca.crt - image: quay.io/project-codeflare/ray:latest-py39-cu118 - imagePullPolicy: Always - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - ray stop - name: ray-head - ports: - - containerPort: 6379 - name: gcs - - containerPort: 8265 - name: dashboard - - containerPort: 10001 - name: client - resources: - limits: - cpu: 2 - memory: 8G - nvidia.com/gpu: 0 - requests: - cpu: 2 - memory: 8G - nvidia.com/gpu: 0 - imagePullSecrets: [] - rayVersion: 2.1.0 - workerGroupSpecs: - - groupName: small-group-instascaletest - maxReplicas: 2 - minReplicas: 2 - rayStartParams: - block: 'true' - num-gpus: '1' - replicas: 2 - template: - metadata: - annotations: - key: value - labels: - key: value - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: instascaletest - operator: In - values: - - instascaletest - containers: - - env: - - name: MY_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: RAY_USE_TLS - value: '0' - - name: RAY_TLS_SERVER_CERT - value: /home/ray/workspace/tls/server.crt - - name: RAY_TLS_SERVER_KEY - value: /home/ray/workspace/tls/server.key - - name: RAY_TLS_CA_CERT - value: /home/ray/workspace/tls/ca.crt - image: quay.io/project-codeflare/ray:latest-py39-cu118 - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - ray stop - name: machine-learning - resources: - limits: - cpu: 2 - memory: 8G - nvidia.com/gpu: 1 - requests: - cpu: 2 - memory: 8G - nvidia.com/gpu: 1 - imagePullSecrets: [] - replicas: 1 - - generictemplate: - apiVersion: route.openshift.io/v1 - kind: Route - metadata: - labels: - odh-ray-cluster-service: instascaletest-head-svc - name: ray-dashboard-instascaletest - namespace: default - spec: - port: - targetPort: dashboard - to: - kind: Service - name: instascaletest-head-svc - replicas: 1 - Items: [] diff --git a/demo-notebooks/guided-demos/preview_nbs/1_basic_instascale.ipynb b/demo-notebooks/guided-demos/preview_nbs/1_basic_instascale.ipynb deleted file mode 100644 index 418737eb6..000000000 --- a/demo-notebooks/guided-demos/preview_nbs/1_basic_instascale.ipynb +++ /dev/null @@ -1,177 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "9865ee8c", - "metadata": {}, - "source": [ - "In this second notebook, we will go over the basics of using InstaScale to scale up/down necessary resources that are not currently available on your OpenShift Cluster (in cloud environments)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b55bc3ea-4ce3-49bf-bb1f-e209de8ca47a", - "metadata": {}, - "outputs": [], - "source": [ - "# Import pieces from codeflare-sdk\n", - "from codeflare_sdk import Cluster, ClusterConfiguration, TokenAuthentication" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "614daa0c", - "metadata": {}, - "outputs": [], - "source": [ - "# Create authentication object for user permissions\n", - "# IF unused, SDK will automatically check for default kubeconfig, then in-cluster config\n", - "# KubeConfigFileAuthentication can also be used to specify kubeconfig path manually\n", - "auth = TokenAuthentication(\n", - " token = \"XXXXX\",\n", - " server = \"XXXXX\",\n", - " skip_tls=False\n", - ")\n", - "auth.login()" - ] - }, - { - "cell_type": "markdown", - "id": "bc27f84c", - "metadata": {}, - "source": [ - "This time, we are working in a cloud environment, and our OpenShift cluster does not have the resources needed for our desired workloads. We will use InstaScale to dynamically scale-up guaranteed resources based on our request (that will also automatically scale-down when we are finished working):\n", - "\n", - "NOTE: We must specify the `image` which will be used in our RayCluster, we recommend you bring your own image which suits your purposes. \n", - "The example here is a community image." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0f4bc870-091f-4e11-9642-cba145710159", - "metadata": {}, - "outputs": [], - "source": [ - "# Create and configure our cluster object (and appwrapper)\n", - "cluster = Cluster(ClusterConfiguration(\n", - " name='instascaletest',\n", - " namespace='default',\n", - " num_workers=2,\n", - " min_cpus=2,\n", - " max_cpus=2,\n", - " min_memory=8,\n", - " max_memory=8,\n", - " num_gpus=1,\n", - " image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n", - " instascale=True, # InstaScale now enabled, will scale OCP cluster to guarantee resource request\n", - " machine_types=[\"m5.xlarge\", \"g4dn.xlarge\"] # Head, worker AWS machine types desired\n", - "))" - ] - }, - { - "cell_type": "markdown", - "id": "12eef53c", - "metadata": {}, - "source": [ - "Same as last time, we will bring the cluster up, wait for it to be ready, and confirm that the specs are as-requested:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f0884bbc-c224-4ca0-98a0-02dfa09c2200", - "metadata": {}, - "outputs": [], - "source": [ - "# Bring up the cluster\n", - "cluster.up()\n", - "cluster.wait_ready()" - ] - }, - { - "cell_type": "markdown", - "id": "6abfe904", - "metadata": {}, - "source": [ - "While the resources are being scaled, we can also go into the console and take a look at the InstaScale logs, as well as the new machines/nodes spinning up.\n", - "\n", - "Once the cluster is ready, we can confirm the specs:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7fd45bc5-03c0-4ae5-9ec5-dd1c30f1a084", - "metadata": {}, - "outputs": [], - "source": [ - "cluster.details()" - ] - }, - { - "cell_type": "markdown", - "id": "5af8cd32", - "metadata": {}, - "source": [ - "Finally, we bring our resource cluster down and release/terminate the associated resources, bringing everything back to the way it was before our cluster was brought up." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5f36db0f-31f6-4373-9503-dc3c1c4c3f57", - "metadata": {}, - "outputs": [], - "source": [ - "cluster.down()" - ] - }, - { - "cell_type": "markdown", - "id": "c883caea", - "metadata": {}, - "source": [ - "Once again, we can look at the machines/nodes and see that everything has been successfully scaled down!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0d41b90e", - "metadata": {}, - "outputs": [], - "source": [ - "auth.logout()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.13" - }, - "vscode": { - "interpreter": { - "hash": "f9f85f796d01129d0dd105a088854619f454435301f6ffec2fea96ecbd9be4ac" - } - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/cluster-configuration.md b/docs/cluster-configuration.md index bb058fa4e..db3a2d248 100644 --- a/docs/cluster-configuration.md +++ b/docs/cluster-configuration.md @@ -20,8 +20,6 @@ cluster = Cluster(ClusterConfiguration( num_gpus=0, # Default 0 mcad=True, # Default True image="quay.io/project-codeflare/ray:latest-py39-cu118", # Mandatory Field - instascale=False, # Default False - machine_types=["m5.xlarge", "g4dn.xlarge"], ingress_domain="example.com" # Default None, Mandatory for Vanilla Kubernetes Clusters - ingress_domain is ignored on OpenShift Clusters as a route is created. local_interactive=False, # Default False )) diff --git a/go.mod b/go.mod index 436ed65c6..c5157495a 100644 --- a/go.mod +++ b/go.mod @@ -1,16 +1,16 @@ module github.com/project-codeflare/codeflare-sdk -go 1.20 +go 1.21 require ( - github.com/onsi/gomega v1.27.10 - github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 - github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 + github.com/onsi/gomega v1.31.1 + github.com/project-codeflare/appwrapper v0.7.1 + github.com/project-codeflare/codeflare-common v0.0.0-20240328182412-38df643db183 github.com/ray-project/kuberay/ray-operator v1.0.0 - k8s.io/api v0.26.3 - k8s.io/apimachinery v0.26.3 + k8s.io/api v0.29.1 + k8s.io/apimachinery v0.29.1 k8s.io/cli-runtime v0.26.3 - k8s.io/client-go v0.26.3 + k8s.io/client-go v0.29.1 k8s.io/kubectl v0.26.3 ) @@ -19,80 +19,85 @@ require ( github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.1.3 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.10.1 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect github.com/fatih/camelcase v1.0.0 // indirect github.com/fvbommel/sortorder v1.0.1 // indirect github.com/go-errors/errors v1.0.1 // indirect - github.com/go-logr/logr v1.2.4 // indirect - github.com/go-openapi/jsonpointer v0.19.6 // indirect - github.com/go-openapi/jsonreference v0.20.1 // indirect - github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-openapi/jsonpointer v0.20.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.4.1 // indirect github.com/golang/glog v1.0.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/btree v1.0.1 // indirect github.com/google/gnostic v0.6.9 // indirect - github.com/google/go-cmp v0.5.9 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/gorilla/css v1.0.0 // indirect + github.com/gorilla/websocket v1.5.0 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect - github.com/imdario/mergo v0.3.12 // indirect - github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/imdario/mergo v0.3.16 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kubeflow/training-operator v1.7.0 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/microcosm-cc/bluemonday v1.0.18 // indirect github.com/mitchellh/go-wordwrap v1.0.0 // indirect github.com/moby/spdystream v0.2.0 // indirect - github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect + github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/openshift-online/ocm-sdk-go v0.1.368 // indirect github.com/openshift/api v0.0.0-20230213134911-7ba313770556 // indirect github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.14.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect + github.com/prometheus/client_golang v1.18.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/spf13/cobra v1.6.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/spf13/cobra v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/xlab/treeprint v1.1.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/net v0.20.0 // indirect + golang.org/x/oauth2 v0.12.0 // indirect + golang.org/x/sys v0.16.0 // indirect + golang.org/x/term v0.16.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/protobuf v1.32.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/component-base v0.26.3 // indirect - k8s.io/klog/v2 v2.100.1 // indirect - k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect - k8s.io/utils v0.0.0-20230505201702-9f6742963106 // indirect - sigs.k8s.io/controller-runtime v0.14.6 // indirect + k8s.io/component-base v0.29.1 // indirect + k8s.io/klog/v2 v2.110.1 // indirect + k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + sigs.k8s.io/controller-runtime v0.17.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/kustomize/api v0.12.1 // indirect sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index 0d6034b87..88042ce2d 100644 --- a/go.sum +++ b/go.sum @@ -46,6 +46,7 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -53,13 +54,15 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= -github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -75,15 +78,14 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= -github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= -github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -92,8 +94,9 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= +github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= @@ -112,23 +115,24 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= -github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= +github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -172,6 +176,8 @@ github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -181,8 +187,9 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -196,18 +203,22 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20230323073829-e72429f035bd h1:r8yyd+DJDmsUhGrRBxH5Pj7KeFK5l+Y3FsgT8keqKtk= +github.com/google/pprof v0.0.0-20230323073829-e72429f035bd/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= @@ -216,10 +227,10 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= -github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw= github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= @@ -293,13 +304,16 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubeflow/training-operator v1.7.0 h1:Zh61GlOWrlRi4UFOtJeV+/5REo/OndhwQ25KYd0llzc= +github.com/kubeflow/training-operator v1.7.0/go.mod h1:BZCLX1h06wY3YSeSZZcGYAqI9/nVi7isVCRkfgZe9nE= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -317,16 +331,16 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= -github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae h1:O4SWKdcHVCvYqyDV+9CJA1fcDN2L11Bule0iFy3YlAI= -github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= +github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA= +github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -340,6 +354,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -348,13 +364,14 @@ github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vv github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= -github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY= +github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo= +github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0= github.com/openshift-online/ocm-sdk-go v0.1.368 h1:qP+gkChV8WDwwpkUw1xUyjTXKdvrwyd70Gff2GMUSeU= github.com/openshift-online/ocm-sdk-go v0.1.368/go.mod h1:KYOw8kAKAHyPrJcQoVR82CneQ4ofC02Na4cXXaTq4Nw= github.com/openshift/api v0.0.0-20230213134911-7ba313770556 h1:7W2fOhJicyEff24VaF7ASNzPtYvr+iSCVft4SIBAzaE= @@ -369,40 +386,42 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 h1:81+ma1mchF/LtAGsf+poAt50kJ/fLYjoTAcZOxci1Yc= -github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 h1:oyhdLdc4BgA4zcH1zlRrSrYpzuVxV5QLDbyIXrwnQqs= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= +github.com/project-codeflare/appwrapper v0.7.1 h1:jnXlYbCgboP3jxPjkqQvmRX3LdoLNyWmYJjbCHwYY+c= +github.com/project-codeflare/appwrapper v0.7.1/go.mod h1:f4SYPB6sPteegtekwD4bjPdpWZ+nFlLF2Eoz9iyw73s= +github.com/project-codeflare/codeflare-common v0.0.0-20240328182412-38df643db183 h1:WOy8nuzZ3zGLVxj2evwfSkTj5SLMVCX5rbI7J5NZxOg= +github.com/project-codeflare/codeflare-common v0.0.0-20240328182412-38df643db183/go.mod h1:ebDcLOZsMe3t9JvUGAv4By2ttu+7NenSpAQZUAaYUak= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/ray-project/kuberay/ray-operator v1.0.0 h1:i69nvbV7az2FG41VHQgxrmhD+SUl8ca+ek4RPbSE2Q0= github.com/ray-project/kuberay/ray-operator v1.0.0/go.mod h1:7C7ebIkxtkmOX8w1iiLrKM1j4hkZs/Guzm3WdePk/yg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= @@ -410,16 +429,18 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI= -github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= @@ -437,7 +458,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= @@ -448,6 +470,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -481,7 +504,8 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -513,6 +537,7 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -553,20 +578,19 @@ golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= +golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -578,6 +602,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -635,13 +660,16 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -651,8 +679,9 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -670,7 +699,6 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -708,7 +736,9 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= -golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= +golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -737,8 +767,8 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -798,8 +828,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -826,8 +856,6 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -835,36 +863,36 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.26.3 h1:emf74GIQMTik01Aum9dPP0gAypL8JTLl/lHa4V9RFSU= -k8s.io/api v0.26.3/go.mod h1:PXsqwPMXBSBcL1lJ9CYDKy7kIReUydukS5JiRlxC3qE= -k8s.io/apimachinery v0.26.3 h1:dQx6PNETJ7nODU3XPtrwkfuubs6w7sX0M8n61zHIV/k= -k8s.io/apimachinery v0.26.3/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I= +k8s.io/api v0.29.1 h1:DAjwWX/9YT7NQD4INu49ROJuZAAAP/Ijki48GUPzxqw= +k8s.io/api v0.29.1/go.mod h1:7Kl10vBRUXhnQQI8YR/R327zXC8eJ7887/+Ybta+RoQ= +k8s.io/apimachinery v0.29.1 h1:KY4/E6km/wLBguvCZv8cKTeOwwOBqFNjwJIdMkMbbRc= +k8s.io/apimachinery v0.29.1/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= k8s.io/cli-runtime v0.26.3 h1:3ULe0oI28xmgeLMVXIstB+ZL5CTGvWSMVMLeHxitIuc= k8s.io/cli-runtime v0.26.3/go.mod h1:5YEhXLV4kLt/OSy9yQwtSSNZU2Z7aTEYta1A+Jg4VC4= -k8s.io/client-go v0.26.3 h1:k1UY+KXfkxV2ScEL3gilKcF7761xkYsSD6BC9szIu8s= -k8s.io/client-go v0.26.3/go.mod h1:ZPNu9lm8/dbRIPAgteN30RSXea6vrCpFvq+MateTUuQ= -k8s.io/component-base v0.26.3 h1:oC0WMK/ggcbGDTkdcqefI4wIZRYdK3JySx9/HADpV0g= -k8s.io/component-base v0.26.3/go.mod h1:5kj1kZYwSC6ZstHJN7oHBqcJC6yyn41eR+Sqa/mQc8E= -k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= -k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= -k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= +k8s.io/client-go v0.29.1 h1:19B/+2NGEwnFLzt0uB5kNJnfTsbV8w6TgQRz9l7ti7A= +k8s.io/client-go v0.29.1/go.mod h1:TDG/psL9hdet0TI9mGyHJSgRkW3H9JZk2dNEUS7bRks= +k8s.io/component-base v0.29.1 h1:MUimqJPCRnnHsskTTjKD+IC1EHBbRCVyi37IoFBrkYw= +k8s.io/component-base v0.29.1/go.mod h1:fP9GFjxYrLERq1GcWWZAE3bqbNcDKDytn2srWuHTtKc= +k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= +k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= k8s.io/kubectl v0.26.3 h1:bZ5SgFyeEXw6XTc1Qji0iNdtqAC76lmeIIQULg2wNXM= k8s.io/kubectl v0.26.3/go.mod h1:02+gv7Qn4dupzN3fi/9OvqqdW+uG/4Zi56vc4Zmsp1g= -k8s.io/utils v0.0.0-20230505201702-9f6742963106 h1:EObNQ3TW2D+WptiYXlApGNLVy0zm/JIBVY9i+M4wpAU= -k8s.io/utils v0.0.0-20230505201702-9f6742963106/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.14.6 h1:oxstGVvXGNnMvY7TAESYk+lzr6S3V5VFxQ6d92KcwQA= -sigs.k8s.io/controller-runtime v0.14.6/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= +sigs.k8s.io/controller-runtime v0.17.0 h1:fjJQf8Ukya+VjogLO6/bNX9HE6Y2xpsO5+fyS26ur/s= +sigs.k8s.io/controller-runtime v0.17.0/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kustomize/api v0.12.1 h1:7YM7gW3kYBwtKvoY216ZzY+8hM+lV53LUayghNRJ0vM= sigs.k8s.io/kustomize/api v0.12.1/go.mod h1:y3JUhimkZkR6sbLNwfJHxvo1TCLwuwm14sCYnkH6S1s= sigs.k8s.io/kustomize/kyaml v0.13.9 h1:Qz53EAaFFANyNgyOEJbT/yoIHygK40/ZcvU3rgry2Tk= sigs.k8s.io/kustomize/kyaml v0.13.9/go.mod h1:QsRbD0/KcU+wdk0/L0fIp2KLnohkVzs6fQ85/nOXac4= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/src/codeflare_sdk/cluster/awload.py b/src/codeflare_sdk/cluster/awload.py index 97d138d5a..3d3f1b8c2 100644 --- a/src/codeflare_sdk/cluster/awload.py +++ b/src/codeflare_sdk/cluster/awload.py @@ -29,8 +29,7 @@ class AWManager: """ - An object for submitting and removing existing AppWrapper yamls - to be added to the MCAD queue. + An object for submitting and removing existing AppWrapper yamls. """ def __init__(self, filename: str) -> None: @@ -62,7 +61,7 @@ def submit(self) -> None: api_instance = client.CustomObjectsApi(api_config_handler()) api_instance.create_namespaced_custom_object( group="workload.codeflare.dev", - version="v1beta1", + version="v1beta2", namespace=self.namespace, plural="appwrappers", body=self.awyaml, @@ -87,7 +86,7 @@ def remove(self) -> None: api_instance = client.CustomObjectsApi(api_config_handler()) api_instance.delete_namespaced_custom_object( group="workload.codeflare.dev", - version="v1beta1", + version="v1beta2", namespace=self.namespace, plural="appwrappers", name=self.name, diff --git a/src/codeflare_sdk/cluster/cluster.py b/src/codeflare_sdk/cluster/cluster.py index 115ac25ec..5ae1ec667 100644 --- a/src/codeflare_sdk/cluster/cluster.py +++ b/src/codeflare_sdk/cluster/cluster.py @@ -106,26 +106,6 @@ def job_client(self): ) return self._job_submission_client - def evaluate_dispatch_priority(self): - priority_class = self.config.dispatch_priority - - try: - config_check() - api_instance = client.CustomObjectsApi(api_config_handler()) - priority_classes = api_instance.list_cluster_custom_object( - group="scheduling.k8s.io", - version="v1", - plural="priorityclasses", - ) - except Exception as e: # pragma: no cover - return _kube_api_error_handling(e) - - for pc in priority_classes["items"]: - if pc["metadata"]["name"] == priority_class: - return pc["value"] - print(f"Priority class {priority_class} is not available in the cluster") - return None - def validate_image_config(self): """ Validates that the image configuration is not empty. @@ -154,20 +134,6 @@ def create_app_wrapper(self): # Validate image configuration self.validate_image_config() - # Before attempting to create the cluster AW, let's evaluate the ClusterConfig - if self.config.dispatch_priority: - if not self.config.mcad: - raise ValueError( - "Invalid Cluster Configuration, cannot have dispatch priority without MCAD" - ) - priority_val = self.evaluate_dispatch_priority() - if priority_val == None: - raise ValueError( - "Invalid Cluster Configuration, AppWrapper not generated" - ) - else: - priority_val = None - name = self.config.name namespace = self.config.namespace head_cpus = self.config.head_cpus @@ -181,13 +147,10 @@ def create_app_wrapper(self): workers = self.config.num_workers template = self.config.template image = self.config.image - instascale = self.config.instascale mcad = self.config.mcad - instance_types = self.config.machine_types env = self.config.envs local_interactive = self.config.local_interactive image_pull_secrets = self.config.image_pull_secrets - dispatch_priority = self.config.dispatch_priority ingress_domain = self.config.ingress_domain ingress_options = self.config.ingress_options write_to_file = self.config.write_to_file @@ -205,14 +168,10 @@ def create_app_wrapper(self): workers=workers, template=template, image=image, - instascale=instascale, mcad=mcad, - instance_types=instance_types, env=env, local_interactive=local_interactive, image_pull_secrets=image_pull_secrets, - dispatch_priority=dispatch_priority, - priority_val=priority_val, openshift_oauth=self.config.openshift_oauth, ingress_domain=ingress_domain, ingress_options=ingress_options, @@ -240,7 +199,7 @@ def up(self): aw = yaml.load(f, Loader=yaml.FullLoader) api_instance.create_namespaced_custom_object( group="workload.codeflare.dev", - version="v1beta1", + version="v1beta2", namespace=namespace, plural="appwrappers", body=aw, @@ -249,7 +208,7 @@ def up(self): aw = yaml.safe_load(self.app_wrapper_yaml) api_instance.create_namespaced_custom_object( group="workload.codeflare.dev", - version="v1beta1", + version="v1beta2", namespace=namespace, plural="appwrappers", body=aw, @@ -271,7 +230,7 @@ def down(self): if self.config.mcad: api_instance.delete_namespaced_custom_object( group="workload.codeflare.dev", - version="v1beta1", + version="v1beta2", namespace=namespace, plural="appwrappers", name=self.app_wrapper_name, @@ -300,25 +259,24 @@ def status( appwrapper = _app_wrapper_status(self.config.name, self.config.namespace) if appwrapper: if appwrapper.status in [ + AppWrapperStatus.RESUMING, AppWrapperStatus.RUNNING, - AppWrapperStatus.COMPLETED, - AppWrapperStatus.RUNNING_HOLD_COMPLETION, + AppWrapperStatus.SUCCEEDED, ]: ready = False status = CodeFlareClusterStatus.STARTING elif appwrapper.status in [ AppWrapperStatus.FAILED, - AppWrapperStatus.DELETED, ]: ready = False status = CodeFlareClusterStatus.FAILED # should deleted be separate return status, ready # exit early, no need to check ray status elif appwrapper.status in [ - AppWrapperStatus.PENDING, - AppWrapperStatus.QUEUEING, + AppWrapperStatus.SUSPENDED, + AppWrapperStatus.SUSPENDING, ]: ready = False - if appwrapper.status == AppWrapperStatus.PENDING: + if appwrapper.status == AppWrapperStatus.SUSPENDED: status = CodeFlareClusterStatus.QUEUED else: status = CodeFlareClusterStatus.QUEUEING @@ -520,11 +478,6 @@ def from_k8_cluster_object( "containers" ]: openshift_oauth = "oauth-proxy" in container["name"] - machine_types = ( - rc["metadata"]["labels"]["orderedinstance"].split("_") - if "orderedinstance" in rc["metadata"]["labels"] - else [] - ) if local_interactive and ingress_domain == None: ingress_domain = rc["metadata"]["annotations"][ @@ -534,7 +487,6 @@ def from_k8_cluster_object( cluster_config = ClusterConfiguration( name=rc["metadata"]["name"], namespace=rc["metadata"]["namespace"], - machine_types=machine_types, num_workers=rc["spec"]["workerGroupSpecs"][0]["minReplicas"], min_cpus=int( rc["spec"]["workerGroupSpecs"][0]["template"]["spec"]["containers"][0][ @@ -561,7 +513,6 @@ def from_k8_cluster_object( "resources" ]["limits"]["nvidia.com/gpu"] ), - instascale=True if machine_types else False, image=rc["spec"]["workerGroupSpecs"][0]["template"]["spec"]["containers"][ 0 ]["image"], @@ -620,9 +571,7 @@ def list_all_queued(namespace: str, print_to_console: bool = True): Returns (and prints by default) a list of all currently queued-up AppWrappers in a given namespace. """ - app_wrappers = _get_app_wrappers( - namespace, filter=[AppWrapperStatus.RUNNING, AppWrapperStatus.PENDING] - ) + app_wrappers = _get_app_wrappers(namespace, filter=[AppWrapperStatus.SUSPENDED]) if print_to_console: pretty_print.print_app_wrappers_status(app_wrappers) return app_wrappers @@ -816,7 +765,7 @@ def _check_aw_exists(name: str, namespace: str) -> bool: api_instance = client.CustomObjectsApi(api_config_handler()) aws = api_instance.list_namespaced_custom_object( group="workload.codeflare.dev", - version="v1beta1", + version="v1beta2", namespace=namespace, plural="appwrappers", ) @@ -876,7 +825,7 @@ def _app_wrapper_status(name, namespace="default") -> Optional[AppWrapper]: api_instance = client.CustomObjectsApi(api_config_handler()) aws = api_instance.list_namespaced_custom_object( group="workload.codeflare.dev", - version="v1beta1", + version="v1beta2", namespace=namespace, plural="appwrappers", ) @@ -937,7 +886,7 @@ def _get_app_wrappers( api_instance = client.CustomObjectsApi(api_config_handler()) aws = api_instance.list_namespaced_custom_object( group="workload.codeflare.dev", - version="v1beta1", + version="v1beta2", namespace=namespace, plural="appwrappers", ) @@ -946,16 +895,13 @@ def _get_app_wrappers( for item in aws["items"]: app_wrapper = _map_to_app_wrapper(item) - if filter and app_wrapper.status in filter: - list_of_app_wrappers.append(app_wrapper) - else: - # Unsure what the purpose of the filter is + if not filter or app_wrapper.status in filter: list_of_app_wrappers.append(app_wrapper) return list_of_app_wrappers def _map_to_ray_cluster(rc) -> Optional[RayCluster]: - if "state" in rc["status"]: + if "status" in rc and "state" in rc["status"]: status = RayClusterStatus(rc["status"]["state"].lower()) else: status = RayClusterStatus.UNKNOWN @@ -1031,18 +977,16 @@ def _map_to_ray_cluster(rc) -> Optional[RayCluster]: def _map_to_app_wrapper(aw) -> AppWrapper: - if "status" in aw and "canrun" in aw["status"]: + if "status" in aw: return AppWrapper( name=aw["metadata"]["name"], - status=AppWrapperStatus(aw["status"]["state"].lower()), - can_run=aw["status"]["canrun"], - job_state=aw["status"]["queuejobstate"], + status=AppWrapperStatus( + aw["status"].get("phase", AppWrapperStatus.SUSPENDED.value).lower() + ), ) return AppWrapper( name=aw["metadata"]["name"], - status=AppWrapperStatus("queueing"), - can_run=False, - job_state="Still adding to queue", + status=AppWrapperStatus.SUSPENDED, ) diff --git a/src/codeflare_sdk/cluster/config.py b/src/codeflare_sdk/cluster/config.py index 86d4252e8..137d6dc48 100644 --- a/src/codeflare_sdk/cluster/config.py +++ b/src/codeflare_sdk/cluster/config.py @@ -37,7 +37,6 @@ class ClusterConfiguration: head_cpus: int = 2 head_memory: int = 8 head_gpus: int = 0 - machine_types: list = field(default_factory=list) # ["m4.xlarge", "g4dn.xlarge"] min_cpus: int = 1 max_cpus: int = 1 num_workers: int = 1 @@ -45,7 +44,6 @@ class ClusterConfiguration: max_memory: int = 2 num_gpus: int = 0 template: str = f"{dir}/templates/base-template.yaml" - instascale: bool = False mcad: bool = True envs: dict = field(default_factory=dict) image: str = "" diff --git a/src/codeflare_sdk/cluster/model.py b/src/codeflare_sdk/cluster/model.py index 2e1abaf75..f4564e3ad 100644 --- a/src/codeflare_sdk/cluster/model.py +++ b/src/codeflare_sdk/cluster/model.py @@ -36,16 +36,16 @@ class RayClusterStatus(Enum): class AppWrapperStatus(Enum): """ - Defines the possible reportable states of an AppWrapper. + Defines the possible reportable phases of an AppWrapper. """ - QUEUEING = "queueing" - PENDING = "pending" + SUSPENDED = "suspended" + RESUMING = "resuming" RUNNING = "running" + SUSPENDING = "suspending" + SUCCEEDED = "succeeded" FAILED = "failed" - DELETED = "deleted" - COMPLETED = "completed" - RUNNING_HOLD_COMPLETION = "runningholdcompletion" + TERMINATING = "terminating" class CodeFlareClusterStatus(Enum): @@ -89,5 +89,3 @@ class AppWrapper: name: str status: AppWrapperStatus - can_run: bool - job_state: str diff --git a/src/codeflare_sdk/templates/base-template.yaml b/src/codeflare_sdk/templates/base-template.yaml index fb6ef4275..856e694b6 100644 --- a/src/codeflare_sdk/templates/base-template.yaml +++ b/src/codeflare_sdk/templates/base-template.yaml @@ -1,38 +1,16 @@ -apiVersion: workload.codeflare.dev/v1beta1 +apiVersion: workload.codeflare.dev/v1beta2 kind: AppWrapper metadata: name: aw-kuberay namespace: default - #new addition - labels: - orderedinstance: "m4.xlarge_g4dn.xlarge" spec: - priority: 9 - resources: - Items: [] - GenericItems: + components: + - podSets: - replicas: 1 - #new addition - custompodresources: - - replicas: 1 - requests: - cpu: 2 - memory: 8G - nvidia.com/gpu: 0 - limits: - cpu: 2 - memory: 8G - nvidia.com/gpu: 0 - - replicas: 3 - requests: - cpu: 2 - memory: 12G - nvidia.com/gpu: 1 - limits: - cpu: 2 - memory: 12G - nvidia.com/gpu: 1 - generictemplate: + path: template.spec.headGroupSpec.template + - replicas: 3 + path: template.spec.workerGroupSpecs[0].template + template: # This config demonstrates KubeRay's Ray autoscaler integration. # The resource requests and limits in this config are too small for production! # For an example with more realistic resource configuration, see @@ -43,7 +21,6 @@ spec: annotations: sdk.codeflare.dev/local_interactive: "False" labels: - workload.codeflare.dev/appwrapper: "aw-kuberay" controller-tools.k8s.io: "1.0" # A unique identifier for the head node and workers of this cluster. name: kuberay-cluster @@ -104,16 +81,6 @@ spec: #pod template template: spec: - #new addition - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: aw-kuberay - operator: In - values: - - "aw-kuberay" containers: # The Ray head pod - env: @@ -240,15 +207,6 @@ spec: # finalizers: # - kubernetes spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: aw-kuberay - operator: In - values: - - "aw-kuberay" initContainers: # the env var $RAY_IP is set by the operator if missing, with the value of the head service name - name: create-cert @@ -338,8 +296,7 @@ spec: - key: odh-ca-bundle.crt path: odh-ca-bundle.crt optional: true - - replicas: 1 - generictemplate: + - template: apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -363,8 +320,7 @@ spec: pathType: Prefix path: / host: ray-dashboard-raytest. - - replicas: 1 - generictemplate: + - template: kind: Route apiVersion: route.openshift.io/v1 metadata: @@ -381,8 +337,7 @@ spec: targetPort: dashboard tls: termination: edge - - replicas: 1 - generictemplate: + - template: apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -405,8 +360,7 @@ spec: path: '' pathType: ImplementationSpecific host: rayclient-raytest. - - replicas: 1 - generictemplate: + - template: apiVersion: route.openshift.io/v1 kind: Route metadata: @@ -423,8 +377,7 @@ spec: to: kind: Service name: deployment-name-head-svc - - replicas: 1 - generictemplate: + - template: apiVersion: v1 data: ca.crt: generated_crt diff --git a/src/codeflare_sdk/utils/generate_yaml.py b/src/codeflare_sdk/utils/generate_yaml.py index da65defd7..dfe45dbcb 100755 --- a/src/codeflare_sdk/utils/generate_yaml.py +++ b/src/codeflare_sdk/utils/generate_yaml.py @@ -68,21 +68,21 @@ def is_openshift_cluster(): def update_dashboard_route(route_item, cluster_name, namespace): - metadata = route_item.get("generictemplate", {}).get("metadata") + metadata = route_item.get("template", {}).get("metadata") metadata["name"] = gen_dashboard_ingress_name(cluster_name) metadata["namespace"] = namespace metadata["labels"]["odh-ray-cluster-service"] = f"{cluster_name}-head-svc" - spec = route_item.get("generictemplate", {}).get("spec") + spec = route_item.get("template", {}).get("spec") spec["to"]["name"] = f"{cluster_name}-head-svc" # ToDo: refactor the update_x_route() functions def update_rayclient_route(route_item, cluster_name, namespace): - metadata = route_item.get("generictemplate", {}).get("metadata") + metadata = route_item.get("template", {}).get("metadata") metadata["name"] = f"rayclient-{cluster_name}" metadata["namespace"] = namespace metadata["labels"]["odh-ray-cluster-service"] = f"{cluster_name}-head-svc" - spec = route_item.get("generictemplate", {}).get("spec") + spec = route_item.get("template", {}).get("spec") spec["to"]["name"] = f"{cluster_name}-head-svc" @@ -111,8 +111,8 @@ def update_rayclient_exposure( def update_dashboard_ingress( ingress_item, cluster_name, namespace, ingress_options, ingress_domain ): # pragma: no cover - metadata = ingress_item.get("generictemplate", {}).get("metadata") - spec = ingress_item.get("generictemplate", {}).get("spec") + metadata = ingress_item.get("template", {}).get("metadata") + spec = ingress_item.get("template", {}).get("spec") if ingress_options != {}: for index, ingress_option in enumerate(ingress_options["ingresses"]): if "ingressName" not in ingress_option.keys(): @@ -194,8 +194,8 @@ def update_dashboard_ingress( def update_rayclient_ingress( ingress_item, cluster_name, namespace, ingress_domain ): # pragma: no cover - metadata = ingress_item.get("generictemplate", {}).get("metadata") - spec = ingress_item.get("generictemplate", {}).get("spec") + metadata = ingress_item.get("template", {}).get("metadata") + spec = ingress_item.get("template", {}).get("spec") metadata["name"] = f"rayclient-{cluster_name}" metadata["namespace"] = namespace metadata["labels"]["odh-ray-cluster-service"] = f"{cluster_name}-head-svc" @@ -223,109 +223,11 @@ def update_names(yaml, item, appwrapper_name, cluster_name, namespace): metadata = yaml.get("metadata") metadata["name"] = appwrapper_name metadata["namespace"] = namespace - lower_meta = item.get("generictemplate", {}).get("metadata") - lower_meta["labels"]["workload.codeflare.dev/appwrapper"] = appwrapper_name + lower_meta = item.get("template", {}).get("metadata") lower_meta["name"] = cluster_name lower_meta["namespace"] = namespace -def update_labels(yaml, instascale, instance_types): - metadata = yaml.get("metadata") - if instascale: - if not len(instance_types) > 0: - sys.exit( - "If instascale is set to true, must provide at least one instance type" - ) - type_str = "" - for type in instance_types: - type_str += type + "_" - type_str = type_str[:-1] - metadata["labels"]["orderedinstance"] = type_str - else: - metadata.pop("labels") - - -def update_priority(yaml, item, dispatch_priority, priority_val): - spec = yaml.get("spec") - if dispatch_priority is not None: - if priority_val: - spec["priority"] = priority_val - else: - raise ValueError( - "AW generation error: Priority value is None, while dispatch_priority is defined" - ) - head = item.get("generictemplate").get("spec").get("headGroupSpec") - worker = item.get("generictemplate").get("spec").get("workerGroupSpecs")[0] - head["template"]["spec"]["priorityClassName"] = dispatch_priority - worker["template"]["spec"]["priorityClassName"] = dispatch_priority - else: - spec.pop("priority") - - -def update_custompodresources( - item, - min_cpu, - max_cpu, - min_memory, - max_memory, - gpu, - workers, - head_cpus, - head_memory, - head_gpus, -): - if "custompodresources" in item.keys(): - custompodresources = item.get("custompodresources") - for i in range(len(custompodresources)): - resource = custompodresources[i] - if i == 0: - # Leave head node resources as template default - resource["requests"]["cpu"] = head_cpus - resource["limits"]["cpu"] = head_cpus - resource["requests"]["memory"] = str(head_memory) + "G" - resource["limits"]["memory"] = str(head_memory) + "G" - resource["requests"]["nvidia.com/gpu"] = head_gpus - resource["limits"]["nvidia.com/gpu"] = head_gpus - - else: - for k, v in resource.items(): - if k == "replicas" and i == 1: - resource[k] = workers - if k == "requests" or k == "limits": - for spec, _ in v.items(): - if spec == "cpu": - if k == "limits": - resource[k][spec] = max_cpu - else: - resource[k][spec] = min_cpu - if spec == "memory": - if k == "limits": - resource[k][spec] = str(max_memory) + "G" - else: - resource[k][spec] = str(min_memory) + "G" - if spec == "nvidia.com/gpu": - if i == 0: - resource[k][spec] = 0 - else: - resource[k][spec] = gpu - else: - sys.exit("Error: malformed template") - - -def update_affinity(spec, appwrapper_name, instascale): - if instascale: - node_selector_terms = ( - spec.get("affinity") - .get("nodeAffinity") - .get("requiredDuringSchedulingIgnoredDuringExecution") - .get("nodeSelectorTerms") - ) - node_selector_terms[0]["matchExpressions"][0]["values"][0] = appwrapper_name - node_selector_terms[0]["matchExpressions"][0]["key"] = appwrapper_name - else: - spec.pop("affinity") - - def update_image(spec, image): containers = spec.get("containers") for container in containers: @@ -374,19 +276,18 @@ def update_nodes( gpu, workers, image, - instascale, env, image_pull_secrets, head_cpus, head_memory, head_gpus, ): - if "generictemplate" in item.keys(): - head = item.get("generictemplate").get("spec").get("headGroupSpec") + if "template" in item.keys(): + head = item.get("template").get("spec").get("headGroupSpec") head["rayStartParams"]["num-gpus"] = str(int(head_gpus)) - worker = item.get("generictemplate").get("spec").get("workerGroupSpecs")[0] - # Head counts as first worker + item.get("podSets")[1]["replicas"] = workers + worker = item.get("template").get("spec").get("workerGroupSpecs")[0] worker["replicas"] = workers worker["minReplicas"] = workers worker["maxReplicas"] = workers @@ -395,7 +296,6 @@ def update_nodes( for comp in [head, worker]: spec = comp.get("template").get("spec") - update_affinity(spec, appwrapper_name, instascale) update_image_pull_secrets(spec, image_pull_secrets) update_image(spec, image) update_env(spec, env) @@ -411,36 +311,36 @@ def update_nodes( def update_ca_secret(ca_secret_item, cluster_name, namespace): from . import generate_cert - metadata = ca_secret_item.get("generictemplate", {}).get("metadata") + metadata = ca_secret_item.get("template", {}).get("metadata") metadata["name"] = f"ca-secret-{cluster_name}" metadata["namespace"] = namespace metadata["labels"]["odh-ray-cluster-service"] = f"{cluster_name}-head-svc" - data = ca_secret_item.get("generictemplate", {}).get("data") + data = ca_secret_item.get("template", {}).get("data") data["ca.key"], data["ca.crt"] = generate_cert.generate_ca_cert(365) -def enable_local_interactive(resources, cluster_name, namespace, ingress_domain): - rayclient_ingress_item = resources["resources"].get("GenericItems")[3] - rayclient_route_item = resources["resources"].get("GenericItems")[4] - ca_secret_item = resources["resources"].get("GenericItems")[5] - item = resources["resources"].get("GenericItems")[0] +def enable_local_interactive(components, cluster_name, namespace, ingress_domain): + rayclient_ingress_item = components[3] + rayclient_route_item = components[4] + ca_secret_item = components[5] + item = components[0] update_ca_secret(ca_secret_item, cluster_name, namespace) # update_ca_secret_volumes - item["generictemplate"]["spec"]["headGroupSpec"]["template"]["spec"]["volumes"][0][ + item["template"]["spec"]["headGroupSpec"]["template"]["spec"]["volumes"][0][ + "secret" + ]["secretName"] = f"ca-secret-{cluster_name}" + item["template"]["spec"]["workerGroupSpecs"][0]["template"]["spec"]["volumes"][0][ "secret" ]["secretName"] = f"ca-secret-{cluster_name}" - item["generictemplate"]["spec"]["workerGroupSpecs"][0]["template"]["spec"][ - "volumes" - ][0]["secret"]["secretName"] = f"ca-secret-{cluster_name}" # update_tls_env - item["generictemplate"]["spec"]["headGroupSpec"]["template"]["spec"]["containers"][ + item["template"]["spec"]["headGroupSpec"]["template"]["spec"]["containers"][0][ + "env" + ][1]["value"] = "1" + item["template"]["spec"]["workerGroupSpecs"][0]["template"]["spec"]["containers"][ 0 ]["env"][1]["value"] = "1" - item["generictemplate"]["spec"]["workerGroupSpecs"][0]["template"]["spec"][ - "containers" - ][0]["env"][1]["value"] = "1" # update_init_container - command = item["generictemplate"]["spec"]["headGroupSpec"]["template"]["spec"][ + command = item["template"]["spec"]["headGroupSpec"]["template"]["spec"][ "initContainers" ][0].get("command")[2] @@ -461,21 +361,21 @@ def enable_local_interactive(resources, cluster_name, namespace, ingress_domain) namespace, ingress_domain, ) - item["generictemplate"]["metadata"]["annotations"][ + item["template"]["metadata"]["annotations"][ "sdk.codeflare.dev/local_interactive" ] = "True" - item["generictemplate"]["metadata"]["annotations"][ + item["template"]["metadata"]["annotations"][ "sdk.codeflare.dev/ingress_domain" ] = ingress_domain - item["generictemplate"]["spec"]["headGroupSpec"]["template"]["spec"][ - "initContainers" - ][0].get("command")[2] = command + item["template"]["spec"]["headGroupSpec"]["template"]["spec"]["initContainers"][ + 0 + ].get("command")[2] = command -def apply_ingress_domain_annotation(resources, ingress_domain): - item = resources["resources"].get("GenericItems")[0] - item["generictemplate"]["metadata"]["annotations"][ +def apply_ingress_domain_annotation(components, ingress_domain): + item = components[0] + item["template"]["metadata"]["annotations"][ "sdk.codeflare.dev/ingress_domain" ] = ingress_domain @@ -484,8 +384,8 @@ def del_from_list_by_name(l: list, target: typing.List[str]) -> list: return [x for x in l if x["name"] not in target] -def disable_raycluster_tls(resources): - generic_template_spec = resources["GenericItems"][0]["generictemplate"]["spec"] +def disable_raycluster_tls(components): + generic_template_spec = components[0]["template"]["spec"] headGroupTemplateSpec = generic_template_spec["headGroupSpec"]["template"]["spec"] headGroupTemplateSpec["volumes"] = del_from_list_by_name( @@ -518,19 +418,20 @@ def disable_raycluster_tls(resources): ] updated_items = [] - for i in resources["GenericItems"][:]: - if "rayclient-deployment-ingress" in i["generictemplate"]["metadata"]["name"]: + for i in components[:]: + if "rayclient-deployment-ingress" in i["template"]["metadata"]["name"]: continue - if "rayclient-deployment-route" in i["generictemplate"]["metadata"]["name"]: + if "rayclient-deployment-route" in i["template"]["metadata"]["name"]: continue - if "ca-secret-deployment-name" in i["generictemplate"]["metadata"]["name"]: + if "ca-secret-deployment-name" in i["template"]["metadata"]["name"]: continue updated_items.append(i) - resources["GenericItems"] = updated_items + components.clear() + components.extend(updated_items) -def delete_route_or_ingress(resources): +def delete_route_or_ingress(components): if is_openshift_cluster(): client_to_remove_name = "rayclient-deployment-ingress" dashboard_to_remove_name = "ray-dashboard-deployment-ingress" @@ -539,15 +440,16 @@ def delete_route_or_ingress(resources): dashboard_to_remove_name = "ray-dashboard-deployment-route" updated_items = [] - for i in resources["GenericItems"][:]: - if dashboard_to_remove_name in i["generictemplate"]["metadata"]["name"]: + for i in components[:]: + if dashboard_to_remove_name in i["template"]["metadata"]["name"]: continue - elif client_to_remove_name in i["generictemplate"]["metadata"]["name"]: + elif client_to_remove_name in i["template"]["metadata"]["name"]: continue updated_items.append(i) - resources["GenericItems"] = updated_items + components.clear() + components.extend(updated_items) def write_user_appwrapper(user_yaml, output_file_name): @@ -588,10 +490,10 @@ def enable_openshift_oauth(user_yaml, cluster_name, namespace): user_yaml["metadata"]["annotations"][ "codeflare-sdk-use-oauth" ] = "true" # if the user gets an - ray_headgroup_pod = user_yaml["spec"]["resources"]["GenericItems"][0][ - "generictemplate" - ]["spec"]["headGroupSpec"]["template"]["spec"] - user_yaml["spec"]["resources"]["GenericItems"].pop(1) + ray_headgroup_pod = user_yaml["spec"]["components"][0]["template"]["spec"][ + "headGroupSpec" + ]["template"]["spec"] + user_yaml["spec"]["components"].pop(1) ray_headgroup_pod["serviceAccount"] = oauth_sa_name ray_headgroup_pod["volumes"] = ray_headgroup_pod.get("volumes", []) @@ -641,24 +543,22 @@ def write_components(user_yaml: dict, output_file_name: str): if not os.path.exists(directory_path): os.makedirs(directory_path) - components = user_yaml.get("spec", "resources")["resources"].get("GenericItems") + components = user_yaml.get("spec").get("components") open(output_file_name, "w").close() with open(output_file_name, "a") as outfile: for component in components: - if "generictemplate" in component: + if "template" in component: outfile.write("---\n") - yaml.dump( - component["generictemplate"], outfile, default_flow_style=False - ) + yaml.dump(component["template"], outfile, default_flow_style=False) print(f"Written to: {output_file_name}") def load_components(user_yaml: dict, name: str): component_list = [] - components = user_yaml.get("spec", "resources")["resources"].get("GenericItems") + components = user_yaml.get("spec").get("components") for component in components: - if "generictemplate" in component: - component_list.append(component["generictemplate"]) + if "template" in component: + component_list.append(component["template"]) resources = "---\n" + "---\n".join( [yaml.dump(component) for component in component_list] @@ -688,14 +588,10 @@ def generate_appwrapper( workers: int, template: str, image: str, - instascale: bool, mcad: bool, - instance_types: list, env, local_interactive: bool, image_pull_secrets: list, - dispatch_priority: str, - priority_val: int, openshift_oauth: bool, ingress_domain: str, ingress_options: dict, @@ -703,25 +599,11 @@ def generate_appwrapper( ): user_yaml = read_template(template) appwrapper_name, cluster_name = gen_names(name) - resources = user_yaml.get("spec", "resources") - item = resources["resources"].get("GenericItems")[0] - ingress_item = resources["resources"].get("GenericItems")[1] - route_item = resources["resources"].get("GenericItems")[2] + components = user_yaml.get("spec").get("components") + item = components[0] + ingress_item = components[1] + route_item = components[2] update_names(user_yaml, item, appwrapper_name, cluster_name, namespace) - update_labels(user_yaml, instascale, instance_types) - update_priority(user_yaml, item, dispatch_priority, priority_val) - update_custompodresources( - item, - min_cpu, - max_cpu, - min_memory, - max_memory, - gpu, - workers, - head_cpus, - head_memory, - head_gpus, - ) update_nodes( item, appwrapper_name, @@ -732,7 +614,6 @@ def generate_appwrapper( gpu, workers, image, - instascale, env, image_pull_secrets, head_cpus, @@ -748,14 +629,14 @@ def generate_appwrapper( ingress_domain, ) if ingress_domain is not None: - apply_ingress_domain_annotation(resources, ingress_domain) + apply_ingress_domain_annotation(components, ingress_domain) if local_interactive: - enable_local_interactive(resources, cluster_name, namespace, ingress_domain) + enable_local_interactive(components, cluster_name, namespace, ingress_domain) else: - disable_raycluster_tls(resources["resources"]) + disable_raycluster_tls(components) - delete_route_or_ingress(resources["resources"]) + delete_route_or_ingress(components) if openshift_oauth: enable_openshift_oauth(user_yaml, cluster_name, namespace) diff --git a/tests/e2e/mnist_raycluster_sdk_oauth_test.py b/tests/e2e/mnist_raycluster_sdk_oauth_test.py index 3e24d4653..55df42ec0 100644 --- a/tests/e2e/mnist_raycluster_sdk_oauth_test.py +++ b/tests/e2e/mnist_raycluster_sdk_oauth_test.py @@ -49,7 +49,6 @@ def run_mnist_raycluster_sdk_oauth(self): min_memory=1, max_memory=2, num_gpus=0, - instascale=False, image=ray_image, openshift_oauth=True, write_to_file=True, @@ -130,7 +129,7 @@ def assert_appwrapper_exists(self): try: self.custom_api.get_namespaced_custom_object( "workload.codeflare.dev", - "v1beta1", + "v1beta2", self.namespace, "appwrappers", "mnist", diff --git a/tests/e2e/mnist_raycluster_sdk_test.py b/tests/e2e/mnist_raycluster_sdk_test.py index 27c1451e0..2195c03ed 100644 --- a/tests/e2e/mnist_raycluster_sdk_test.py +++ b/tests/e2e/mnist_raycluster_sdk_test.py @@ -67,7 +67,6 @@ def run_mnist_raycluster_sdk(self): min_memory=1, max_memory=2, num_gpus=0, - instascale=False, image=ray_image, ingress_options=ingress_options, write_to_file=True, @@ -119,7 +118,7 @@ def assert_appwrapper_exists(self): try: self.custom_api.get_namespaced_custom_object( "workload.codeflare.dev", - "v1beta1", + "v1beta2", self.namespace, "appwrappers", "mnist", diff --git a/tests/e2e/start_ray_cluster.py b/tests/e2e/start_ray_cluster.py index 774be8f04..c8fbf4bc7 100644 --- a/tests/e2e/start_ray_cluster.py +++ b/tests/e2e/start_ray_cluster.py @@ -35,7 +35,6 @@ min_memory=1, max_memory=2, num_gpus=0, - instascale=False, image=ray_image, ingress_options=ingress_options, ) diff --git a/tests/test-case-bad.yaml b/tests/test-case-bad.yaml index aeccf5194..629b4a387 100644 --- a/tests/test-case-bad.yaml +++ b/tests/test-case-bad.yaml @@ -1,4 +1,4 @@ -apiVersion: workload.codeflare.dev/v1beta1 +apiVersion: workload.codeflare.dev/v1beta2 kind: AppsWrapper metadata: labels: @@ -6,29 +6,13 @@ metadata: nam: unit-test-cluster namspace: ns spec: - priority: 9 - resources: - GenericItems: - - custompodresources: - - limits: - cpu: 2 - memory: 8G - nvidia.com/gpu: 0 - replicas: 1 - requests: - cpu: 2 - memory: 8G - nvidia.com/gpu: 0 - - limits: - cpu: 4 - memory: 6G - nvidia.com/gpu: 7 - replicas: 2 - requests: - cpu: 3 - memory: 5G - nvidia.com/gpu: 7 - generictemplate: + components: + - podSets: + - replicas: 1 + path: template.spec.headGroupSpec.template + - replicas: 2 + path: template.spec.workerGroupSpecs[0].template + template: apiVersion: ray.io/v1 kind: RayCluster metadata: @@ -150,8 +134,7 @@ spec: cpu: 3 memory: 5G nvidia.com/gpu: 7 - replicas: 1 - - generictemplate: + - template: apiVersion: route.openshift.io/v1 kind: Route metadata: @@ -165,5 +148,3 @@ spec: to: kind: Service name: unit-test-cluster-head-svc - replicas: 1 - Items: [] diff --git a/tests/test-case-no-mcad.yamls b/tests/test-case-no-mcad.yamls index b15833fea..a31ca675e 100644 --- a/tests/test-case-no-mcad.yamls +++ b/tests/test-case-no-mcad.yamls @@ -7,7 +7,6 @@ metadata: sdk.codeflare.dev/local_interactive: 'False' labels: controller-tools.k8s.io: '1.0' - workload.codeflare.dev/appwrapper: unit-test-cluster-ray name: unit-test-cluster-ray namespace: ns spec: @@ -31,15 +30,6 @@ spec: serviceType: ClusterIP template: spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: unit-test-cluster-ray - operator: In - values: - - unit-test-cluster-ray containers: - env: - name: MY_POD_IP @@ -126,15 +116,6 @@ spec: labels: key: value spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: unit-test-cluster-ray - operator: In - values: - - unit-test-cluster-ray containers: - env: - name: MY_POD_IP diff --git a/tests/test-case-prio.yaml b/tests/test-case-prio.yaml deleted file mode 100644 index c81d43969..000000000 --- a/tests/test-case-prio.yaml +++ /dev/null @@ -1,256 +0,0 @@ -apiVersion: workload.codeflare.dev/v1beta1 -kind: AppWrapper -metadata: - labels: - orderedinstance: cpu.small_gpu.large - name: prio-test-cluster - namespace: ns -spec: - priority: 10 - resources: - GenericItems: - - custompodresources: - - limits: - cpu: 2 - memory: 8G - nvidia.com/gpu: 0 - replicas: 1 - requests: - cpu: 2 - memory: 8G - nvidia.com/gpu: 0 - - limits: - cpu: 4 - memory: 6G - nvidia.com/gpu: 7 - replicas: 2 - requests: - cpu: 3 - memory: 5G - nvidia.com/gpu: 7 - generictemplate: - apiVersion: ray.io/v1 - kind: RayCluster - metadata: - annotations: - sdk.codeflare.dev/ingress_domain: apps.cluster.awsroute.org - sdk.codeflare.dev/local_interactive: 'False' - labels: - controller-tools.k8s.io: '1.0' - workload.codeflare.dev/appwrapper: prio-test-cluster - name: prio-test-cluster - namespace: ns - spec: - autoscalerOptions: - idleTimeoutSeconds: 60 - imagePullPolicy: Always - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 500m - memory: 512Mi - upscalingMode: Default - enableInTreeAutoscaling: false - headGroupSpec: - rayStartParams: - block: 'true' - dashboard-host: 0.0.0.0 - num-gpus: '0' - serviceType: ClusterIP - template: - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: prio-test-cluster - operator: In - values: - - prio-test-cluster - containers: - - env: - - name: MY_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: RAY_USE_TLS - value: '0' - - name: RAY_TLS_SERVER_CERT - value: /home/ray/workspace/tls/server.crt - - name: RAY_TLS_SERVER_KEY - value: /home/ray/workspace/tls/server.key - - name: RAY_TLS_CA_CERT - value: /home/ray/workspace/tls/ca.crt - image: quay.io/project-codeflare/ray:latest-py39-cu118 - imagePullPolicy: Always - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - ray stop - name: ray-head - ports: - - containerPort: 6379 - name: gcs - - containerPort: 8265 - name: dashboard - - containerPort: 10001 - name: client - resources: - limits: - cpu: 2 - memory: 8G - nvidia.com/gpu: 0 - requests: - cpu: 2 - memory: 8G - nvidia.com/gpu: 0 - volumeMounts: - - mountPath: /etc/pki/tls/certs/odh-trusted-ca-bundle.crt - name: odh-trusted-ca-cert - subPath: odh-trusted-ca-bundle.crt - - mountPath: /etc/ssl/certs/odh-trusted-ca-bundle.crt - name: odh-trusted-ca-cert - subPath: odh-trusted-ca-bundle.crt - - mountPath: /etc/pki/tls/certs/odh-ca-bundle.crt - name: odh-ca-cert - subPath: odh-ca-bundle.crt - - mountPath: /etc/ssl/certs/odh-ca-bundle.crt - name: odh-ca-cert - subPath: odh-ca-bundle.crt - imagePullSecrets: - - name: unit-test-pull-secret - priorityClassName: default - volumes: - - configMap: - items: - - key: ca-bundle.crt - path: odh-trusted-ca-bundle.crt - name: odh-trusted-ca-bundle - optional: true - name: odh-trusted-ca-cert - - configMap: - items: - - key: odh-ca-bundle.crt - path: odh-ca-bundle.crt - name: odh-trusted-ca-bundle - optional: true - name: odh-ca-cert - rayVersion: 2.7.0 - workerGroupSpecs: - - groupName: small-group-prio-test-cluster - maxReplicas: 2 - minReplicas: 2 - rayStartParams: - block: 'true' - num-gpus: '7' - replicas: 2 - template: - metadata: - annotations: - key: value - labels: - key: value - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: prio-test-cluster - operator: In - values: - - prio-test-cluster - containers: - - env: - - name: MY_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: RAY_USE_TLS - value: '0' - - name: RAY_TLS_SERVER_CERT - value: /home/ray/workspace/tls/server.crt - - name: RAY_TLS_SERVER_KEY - value: /home/ray/workspace/tls/server.key - - name: RAY_TLS_CA_CERT - value: /home/ray/workspace/tls/ca.crt - image: quay.io/project-codeflare/ray:latest-py39-cu118 - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - ray stop - name: machine-learning - resources: - limits: - cpu: 4 - memory: 6G - nvidia.com/gpu: 7 - requests: - cpu: 3 - memory: 5G - nvidia.com/gpu: 7 - volumeMounts: - - mountPath: /etc/pki/tls/certs/odh-trusted-ca-bundle.crt - name: odh-trusted-ca-cert - subPath: odh-trusted-ca-bundle.crt - - mountPath: /etc/ssl/certs/odh-trusted-ca-bundle.crt - name: odh-trusted-ca-cert - subPath: odh-trusted-ca-bundle.crt - - mountPath: /etc/pki/tls/certs/odh-ca-bundle.crt - name: odh-ca-cert - subPath: odh-ca-bundle.crt - - mountPath: /etc/ssl/certs/odh-ca-bundle.crt - name: odh-ca-cert - subPath: odh-ca-bundle.crt - imagePullSecrets: - - name: unit-test-pull-secret - priorityClassName: default - volumes: - - configMap: - items: - - key: ca-bundle.crt - path: odh-trusted-ca-bundle.crt - name: odh-trusted-ca-bundle - optional: true - name: odh-trusted-ca-cert - - configMap: - items: - - key: odh-ca-bundle.crt - path: odh-ca-bundle.crt - name: odh-trusted-ca-bundle - optional: true - name: odh-ca-cert - replicas: 1 - - generictemplate: - apiVersion: networking.k8s.io/v1 - kind: Ingress - metadata: - labels: - ingress-options: 'false' - ingress-owner: prio-test-cluster - name: ray-dashboard-prio-test-cluster - namespace: ns - spec: - ingressClassName: nginx - rules: - - host: ray-dashboard-prio-test-cluster-ns.apps.cluster.awsroute.org - http: - paths: - - backend: - service: - name: prio-test-cluster-head-svc - port: - number: 8265 - path: / - pathType: Prefix - replicas: 1 - Items: [] diff --git a/tests/test-case.yaml b/tests/test-case.yaml index d7c31a11a..0713f1080 100644 --- a/tests/test-case.yaml +++ b/tests/test-case.yaml @@ -1,253 +1,214 @@ -apiVersion: workload.codeflare.dev/v1beta1 +apiVersion: workload.codeflare.dev/v1beta2 kind: AppWrapper metadata: - labels: - orderedinstance: cpu.small_gpu.large name: unit-test-cluster namespace: ns spec: - resources: - GenericItems: - - custompodresources: - - limits: - cpu: 2 - memory: 8G - nvidia.com/gpu: 0 - replicas: 1 - requests: - cpu: 2 - memory: 8G - nvidia.com/gpu: 0 - - limits: - cpu: 4 - memory: 6G - nvidia.com/gpu: 7 - replicas: 2 - requests: - cpu: 3 - memory: 5G - nvidia.com/gpu: 7 - generictemplate: - apiVersion: ray.io/v1 - kind: RayCluster - metadata: - annotations: - sdk.codeflare.dev/ingress_domain: apps.cluster.awsroute.org - sdk.codeflare.dev/local_interactive: 'False' - labels: - controller-tools.k8s.io: '1.0' - workload.codeflare.dev/appwrapper: unit-test-cluster - name: unit-test-cluster - namespace: ns - spec: - autoscalerOptions: - idleTimeoutSeconds: 60 - imagePullPolicy: Always - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 500m - memory: 512Mi - upscalingMode: Default - enableInTreeAutoscaling: false - headGroupSpec: - rayStartParams: - block: 'true' - dashboard-host: 0.0.0.0 - num-gpus: '0' - serviceType: ClusterIP - template: - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: unit-test-cluster - operator: In - values: - - unit-test-cluster - containers: - - env: - - name: MY_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: RAY_USE_TLS - value: '0' - - name: RAY_TLS_SERVER_CERT - value: /home/ray/workspace/tls/server.crt - - name: RAY_TLS_SERVER_KEY - value: /home/ray/workspace/tls/server.key - - name: RAY_TLS_CA_CERT - value: /home/ray/workspace/tls/ca.crt - image: quay.io/project-codeflare/ray:latest-py39-cu118 - imagePullPolicy: Always - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - ray stop - name: ray-head - ports: - - containerPort: 6379 - name: gcs - - containerPort: 8265 - name: dashboard - - containerPort: 10001 - name: client - resources: - limits: - cpu: 2 - memory: 8G - nvidia.com/gpu: 0 - requests: - cpu: 2 - memory: 8G - nvidia.com/gpu: 0 - volumeMounts: - - mountPath: /etc/pki/tls/certs/odh-trusted-ca-bundle.crt - name: odh-trusted-ca-cert - subPath: odh-trusted-ca-bundle.crt - - mountPath: /etc/ssl/certs/odh-trusted-ca-bundle.crt - name: odh-trusted-ca-cert - subPath: odh-trusted-ca-bundle.crt - - mountPath: /etc/pki/tls/certs/odh-ca-bundle.crt - name: odh-ca-cert - subPath: odh-ca-bundle.crt - - mountPath: /etc/ssl/certs/odh-ca-bundle.crt - name: odh-ca-cert - subPath: odh-ca-bundle.crt - imagePullSecrets: - - name: unit-test-pull-secret - volumes: - - configMap: - items: - - key: ca-bundle.crt - path: odh-trusted-ca-bundle.crt - name: odh-trusted-ca-bundle - optional: true + components: + - podSets: + - path: template.spec.headGroupSpec.template + replicas: 1 + - path: template.spec.workerGroupSpecs[0].template + replicas: 2 + template: + apiVersion: ray.io/v1 + kind: RayCluster + metadata: + annotations: + sdk.codeflare.dev/ingress_domain: apps.cluster.awsroute.org + sdk.codeflare.dev/local_interactive: 'False' + labels: + controller-tools.k8s.io: '1.0' + name: unit-test-cluster + namespace: ns + spec: + autoscalerOptions: + idleTimeoutSeconds: 60 + imagePullPolicy: Always + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 500m + memory: 512Mi + upscalingMode: Default + enableInTreeAutoscaling: false + headGroupSpec: + rayStartParams: + block: 'true' + dashboard-host: 0.0.0.0 + num-gpus: '0' + serviceType: ClusterIP + template: + spec: + containers: + - env: + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: RAY_USE_TLS + value: '0' + - name: RAY_TLS_SERVER_CERT + value: /home/ray/workspace/tls/server.crt + - name: RAY_TLS_SERVER_KEY + value: /home/ray/workspace/tls/server.key + - name: RAY_TLS_CA_CERT + value: /home/ray/workspace/tls/ca.crt + image: quay.io/project-codeflare/ray:latest-py39-cu118 + imagePullPolicy: Always + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - ray stop + name: ray-head + ports: + - containerPort: 6379 + name: gcs + - containerPort: 8265 + name: dashboard + - containerPort: 10001 + name: client + resources: + limits: + cpu: 2 + memory: 8G + nvidia.com/gpu: 0 + requests: + cpu: 2 + memory: 8G + nvidia.com/gpu: 0 + volumeMounts: + - mountPath: /etc/pki/tls/certs/odh-trusted-ca-bundle.crt + name: odh-trusted-ca-cert + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-trusted-ca-bundle.crt name: odh-trusted-ca-cert - - configMap: - items: - - key: odh-ca-bundle.crt - path: odh-ca-bundle.crt - name: odh-trusted-ca-bundle - optional: true + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/pki/tls/certs/odh-ca-bundle.crt name: odh-ca-cert - rayVersion: 2.7.0 - workerGroupSpecs: - - groupName: small-group-unit-test-cluster - maxReplicas: 2 - minReplicas: 2 - rayStartParams: - block: 'true' - num-gpus: '7' - replicas: 2 - template: - metadata: - annotations: - key: value - labels: - key: value - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: unit-test-cluster - operator: In - values: - - unit-test-cluster - containers: - - env: - - name: MY_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: RAY_USE_TLS - value: '0' - - name: RAY_TLS_SERVER_CERT - value: /home/ray/workspace/tls/server.crt - - name: RAY_TLS_SERVER_KEY - value: /home/ray/workspace/tls/server.key - - name: RAY_TLS_CA_CERT - value: /home/ray/workspace/tls/ca.crt - image: quay.io/project-codeflare/ray:latest-py39-cu118 - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - ray stop - name: machine-learning - resources: - limits: - cpu: 4 - memory: 6G - nvidia.com/gpu: 7 - requests: - cpu: 3 - memory: 5G - nvidia.com/gpu: 7 - volumeMounts: - - mountPath: /etc/pki/tls/certs/odh-trusted-ca-bundle.crt - name: odh-trusted-ca-cert - subPath: odh-trusted-ca-bundle.crt - - mountPath: /etc/ssl/certs/odh-trusted-ca-bundle.crt - name: odh-trusted-ca-cert - subPath: odh-trusted-ca-bundle.crt - - mountPath: /etc/pki/tls/certs/odh-ca-bundle.crt - name: odh-ca-cert - subPath: odh-ca-bundle.crt - - mountPath: /etc/ssl/certs/odh-ca-bundle.crt - name: odh-ca-cert - subPath: odh-ca-bundle.crt - imagePullSecrets: - - name: unit-test-pull-secret - volumes: - - configMap: - items: - - key: ca-bundle.crt - path: odh-trusted-ca-bundle.crt - name: odh-trusted-ca-bundle - optional: true + subPath: odh-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-ca-bundle.crt + name: odh-ca-cert + subPath: odh-ca-bundle.crt + imagePullSecrets: + - name: unit-test-pull-secret + volumes: + - configMap: + items: + - key: ca-bundle.crt + path: odh-trusted-ca-bundle.crt + name: odh-trusted-ca-bundle + optional: true + name: odh-trusted-ca-cert + - configMap: + items: + - key: odh-ca-bundle.crt + path: odh-ca-bundle.crt + name: odh-trusted-ca-bundle + optional: true + name: odh-ca-cert + rayVersion: 2.7.0 + workerGroupSpecs: + - groupName: small-group-unit-test-cluster + maxReplicas: 2 + minReplicas: 2 + rayStartParams: + block: 'true' + num-gpus: '7' + replicas: 2 + template: + metadata: + annotations: + key: value + labels: + key: value + spec: + containers: + - env: + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: RAY_USE_TLS + value: '0' + - name: RAY_TLS_SERVER_CERT + value: /home/ray/workspace/tls/server.crt + - name: RAY_TLS_SERVER_KEY + value: /home/ray/workspace/tls/server.key + - name: RAY_TLS_CA_CERT + value: /home/ray/workspace/tls/ca.crt + image: quay.io/project-codeflare/ray:latest-py39-cu118 + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - ray stop + name: machine-learning + resources: + limits: + cpu: 4 + memory: 6G + nvidia.com/gpu: 7 + requests: + cpu: 3 + memory: 5G + nvidia.com/gpu: 7 + volumeMounts: + - mountPath: /etc/pki/tls/certs/odh-trusted-ca-bundle.crt + name: odh-trusted-ca-cert + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-trusted-ca-bundle.crt name: odh-trusted-ca-cert - - configMap: - items: - - key: odh-ca-bundle.crt - path: odh-ca-bundle.crt - name: odh-trusted-ca-bundle - optional: true + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/pki/tls/certs/odh-ca-bundle.crt name: odh-ca-cert - replicas: 1 - - generictemplate: - apiVersion: networking.k8s.io/v1 - kind: Ingress - metadata: - labels: - ingress-options: 'false' - ingress-owner: unit-test-cluster - name: ray-dashboard-unit-test-cluster - namespace: ns - spec: - ingressClassName: nginx - rules: - - host: ray-dashboard-unit-test-cluster-ns.apps.cluster.awsroute.org - http: - paths: - - backend: - service: - name: unit-test-cluster-head-svc - port: - number: 8265 - path: / - pathType: Prefix - replicas: 1 - Items: [] + subPath: odh-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-ca-bundle.crt + name: odh-ca-cert + subPath: odh-ca-bundle.crt + imagePullSecrets: + - name: unit-test-pull-secret + volumes: + - configMap: + items: + - key: ca-bundle.crt + path: odh-trusted-ca-bundle.crt + name: odh-trusted-ca-bundle + optional: true + name: odh-trusted-ca-cert + - configMap: + items: + - key: odh-ca-bundle.crt + path: odh-ca-bundle.crt + name: odh-trusted-ca-bundle + optional: true + name: odh-ca-cert + - template: + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + labels: + ingress-options: 'false' + ingress-owner: unit-test-cluster + name: ray-dashboard-unit-test-cluster + namespace: ns + spec: + ingressClassName: nginx + rules: + - host: ray-dashboard-unit-test-cluster-ns.apps.cluster.awsroute.org + http: + paths: + - backend: + service: + name: unit-test-cluster-head-svc + port: + number: 8265 + path: / + pathType: Prefix diff --git a/tests/test-default-appwrapper.yaml b/tests/test-default-appwrapper.yaml index c9da340cf..5b74572fa 100644 --- a/tests/test-default-appwrapper.yaml +++ b/tests/test-default-appwrapper.yaml @@ -1,31 +1,16 @@ -apiVersion: workload.codeflare.dev/v1beta1 +apiVersion: workload.codeflare.dev/v1beta2 kind: AppWrapper metadata: name: unit-test-default-cluster namespace: opendatahub spec: - resources: - GenericItems: - - custompodresources: - - limits: - cpu: 2 - memory: 8G - nvidia.com/gpu: 0 - replicas: 1 - requests: - cpu: 2 - memory: 8G - nvidia.com/gpu: 0 - - limits: - cpu: 1 - memory: 2G - nvidia.com/gpu: 0 - replicas: 1 - requests: - cpu: 1 - memory: 2G - nvidia.com/gpu: 0 - generictemplate: + components: + - podSets: + - path: template.spec.headGroupSpec.template + replicas: 1 + - path: template.spec.workerGroupSpecs[0].template + replicas: 1 + template: apiVersion: ray.io/v1 kind: RayCluster metadata: @@ -34,7 +19,6 @@ spec: sdk.codeflare.dev/local_interactive: 'False' labels: controller-tools.k8s.io: '1.0' - workload.codeflare.dev/appwrapper: unit-test-default-cluster name: unit-test-default-cluster namespace: opendatahub spec: @@ -204,8 +188,7 @@ spec: name: odh-trusted-ca-bundle optional: true name: odh-ca-cert - replicas: 1 - - generictemplate: + - template: apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -227,5 +210,3 @@ spec: number: 8265 path: / pathType: Prefix - replicas: 1 - Items: [] diff --git a/tests/unit_test.py b/tests/unit_test.py index 3edadc63c..86a28f6d4 100644 --- a/tests/unit_test.py +++ b/tests/unit_test.py @@ -273,10 +273,7 @@ def test_config_creation(): assert config.num_gpus == 7 assert config.image == "quay.io/project-codeflare/ray:latest-py39-cu118" assert config.template == f"{parent}/src/codeflare_sdk/templates/base-template.yaml" - assert config.instascale - assert config.machine_types == ["cpu.small", "gpu.large"] assert config.image_pull_secrets == ["unit-test-pull-secret"] - assert config.dispatch_priority == None assert config.mcad == True assert config.local_interactive == False @@ -335,8 +332,6 @@ def test_cluster_creation_no_mcad(mocker): min_memory=5, max_memory=6, num_gpus=7, - instascale=True, - machine_types=["cpu.small", "gpu.large"], image_pull_secrets=["unit-test-pull-secret"], ingress_domain="apps.cluster.awsroute.org", image="quay.io/project-codeflare/ray:latest-py39-cu118", @@ -358,30 +353,6 @@ def test_cluster_creation_no_mcad(mocker): assert test_resources == expected_resources -def test_cluster_creation_priority(mocker): - mocker.patch("kubernetes.client.ApisApi.get_api_versions") - mocker.patch("kubernetes.config.load_kube_config", return_value="ignore") - mocker.patch( - "kubernetes.client.CustomObjectsApi.list_cluster_custom_object", - return_value={"items": [{"metadata": {"name": "default"}, "value": 10}]}, - ) - config = createClusterConfig() - config.name = "prio-test-cluster" - config.dispatch_priority = "default" - mocker.patch( - "kubernetes.client.CustomObjectsApi.get_cluster_custom_object", - return_value={"spec": {"domain": "apps.cluster.awsroute.org"}}, - ) - cluster = Cluster(config) - assert cluster.app_wrapper_yaml == f"{aw_dir}prio-test-cluster.yaml" - assert cluster.app_wrapper_name == "prio-test-cluster" - assert filecmp.cmp( - f"{aw_dir}prio-test-cluster.yaml", - f"{parent}/tests/test-case-prio.yaml", - shallow=True, - ) - - def test_default_cluster_creation(mocker): mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch( @@ -429,7 +400,7 @@ def arg_check_apply_effect(group, version, namespace, plural, body, *args): assert args == tuple() if plural == "appwrappers": assert group == "workload.codeflare.dev" - assert version == "v1beta1" + assert version == "v1beta2" with open(f"{aw_dir}unit-test-cluster.yaml") as f: aw = yaml.load(f, Loader=yaml.FullLoader) assert body == aw @@ -466,7 +437,7 @@ def arg_check_del_effect(group, version, namespace, plural, name, *args): assert args == tuple() if plural == "appwrappers": assert group == "workload.codeflare.dev" - assert version == "v1beta1" + assert version == "v1beta2" assert name == "unit-test-cluster" elif plural == "rayclusters": assert group == "ray.io" @@ -552,7 +523,7 @@ def test_get_ingress_domain(mocker): def aw_status_fields(group, version, namespace, plural, *args): assert group == "workload.codeflare.dev" - assert version == "v1beta1" + assert version == "v1beta2" assert namespace == "test-ns" assert plural == "appwrappers" assert args == tuple() @@ -798,15 +769,11 @@ def test_print_no_cluster(capsys): def test_print_appwrappers(capsys): aw1 = AppWrapper( name="awtest1", - status=AppWrapperStatus.PENDING, - can_run=False, - job_state="queue-state", + status=AppWrapperStatus.SUSPENDED, ) aw2 = AppWrapper( name="awtest2", status=AppWrapperStatus.RUNNING, - can_run=False, - job_state="queue-state", ) try: print_app_wrappers_status([aw1, aw2]) @@ -814,18 +781,18 @@ def test_print_appwrappers(capsys): assert 1 == 0 captured = capsys.readouterr() assert captured.out == ( - "โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ\n" - "โ”‚ ๐Ÿš€ Cluster Queue โ”‚\n" - "โ”‚ Status ๐Ÿš€ โ”‚\n" - "โ”‚ +---------+---------+ โ”‚\n" - "โ”‚ | Name | Status | โ”‚\n" - "โ”‚ +=========+=========+ โ”‚\n" - "โ”‚ | awtest1 | pending | โ”‚\n" - "โ”‚ | | | โ”‚\n" - "โ”‚ | awtest2 | running | โ”‚\n" - "โ”‚ | | | โ”‚\n" - "โ”‚ +---------+---------+ โ”‚\n" - "โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ\n" + "โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ\n" + "โ”‚ ๐Ÿš€ Cluster Queue โ”‚\n" + "โ”‚ Status ๐Ÿš€ โ”‚\n" + "โ”‚ +---------+-----------+ โ”‚\n" + "โ”‚ | Name | Status | โ”‚\n" + "โ”‚ +=========+===========+ โ”‚\n" + "โ”‚ | awtest1 | suspended | โ”‚\n" + "โ”‚ | | | โ”‚\n" + "โ”‚ | awtest2 | running | โ”‚\n" + "โ”‚ | | | โ”‚\n" + "โ”‚ +---------+-----------+ โ”‚\n" + "โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ\n" ) @@ -1092,7 +1059,7 @@ def get_ray_obj(group, version, namespace, plural, cls=None): "namespace": "ns", "ownerReferences": [ { - "apiVersion": "workload.codeflare.dev/v1beta1", + "apiVersion": "workload.codeflare.dev/v1beta2", "blockOwnerDeletion": True, "controller": True, "kind": "AppWrapper", @@ -1425,7 +1392,7 @@ def get_ray_obj(group, version, namespace, plural, cls=None): def get_named_aw(group, version, namespace, plural, name): - aws = get_aw_obj("workload.codeflare.dev", "v1beta1", "ns", "appwrappers") + aws = get_aw_obj("workload.codeflare.dev", "v1beta2", "ns", "appwrappers") return aws["items"][0] @@ -1433,37 +1400,27 @@ def get_aw_obj(group, version, namespace, plural): api_obj1 = { "items": [ { - "apiVersion": "workload.codeflare.dev/v1beta1", + "apiVersion": "workload.codeflare.dev/v1beta2", "kind": "AppWrapper", "metadata": { "annotations": { - "kubectl.kubernetes.io/last-applied-configuration": '{"apiVersion":"codeflare.dev/v1beta1","kind":"AppWrapper","metadata":{"annotations":{},"name":"quicktest1","namespace":"ns"},"spec":{"priority":9,"resources":{"GenericItems":[{"custompodresources":[{"limits":{"cpu":2,"memory":"8G","nvidia.com/gpu":0},"replicas":1,"requests":{"cpu":2,"memory":"8G","nvidia.com/gpu":0}},{"limits":{"cpu":1,"memory":"2G","nvidia.com/gpu":0},"replicas":1,"requests":{"cpu":1,"memory":"2G","nvidia.com/gpu":0}}],"generictemplate":{"apiVersion":"ray.io/v1","kind":"RayCluster","metadata":{"labels":{"appwrapper.codeflare.dev":"quicktest1","controller-tools.k8s.io":"1.0"},"name":"quicktest1","namespace":"ns"},"spec":{"autoscalerOptions":{"idleTimeoutSeconds":60,"imagePullPolicy":"Always","resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"500m","memory":"512Mi"}},"upscalingMode":"Default"},"enableInTreeAutoscaling":false,"headGroupSpec":{"rayStartParams":{"block":"true","dashboard-host":"0.0.0.0","num-gpus":"0"},"serviceType":"ClusterIP","template":{"spec":{"containers":[{"image":"ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103","imagePullPolicy":"Always","lifecycle":{"preStop":{"exec":{"command":["/bin/sh","-c","ray stop"]}}},"name":"ray-head","ports":[{"containerPort":6379,"name":"gcs"},{"containerPort":8265,"name":"dashboard"},{"containerPort":10001,"name":"client"}],"resources":{"limits":{"cpu":2,"memory":"8G","nvidia.com/gpu":0},"requests":{"cpu":2,"memory":"8G","nvidia.com/gpu":0}}}]}}},"rayVersion":"1.12.0","workerGroupSpecs":[{"groupName":"small-group-quicktest","maxReplicas":1,"minReplicas":1,"rayStartParams":{"block":"true","num-gpus":"0"},"replicas":1,"template":{"metadata":{"annotations":{"key":"value"},"labels":{"key":"value"}},"spec":{"containers":[{"env":[{"name":"MY_POD_IP","valueFrom":{"fieldRef":{"fieldPath":"status.podIP"}}}],"image":"ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103","lifecycle":{"preStop":{"exec":{"command":["/bin/sh","-c","ray stop"]}}},"name":"machine-learning","resources":{"limits":{"cpu":1,"memory":"2G","nvidia.com/gpu":0},"requests":{"cpu":1,"memory":"2G","nvidia.com/gpu":0}}}],}}}]}},"replicas":1},{"generictemplate":{"apiVersion":"route.openshift.io/v1","kind":"Route","metadata":{"labels":{"odh-ray-cluster-service":"quicktest-head-svc"},"name":"ray-dashboard-quicktest","namespace":"default"},"spec":{"port":{"targetPort":"dashboard"},"to":{"kind":"Service","name":"quicktest-head-svc"}}},"replica":1}],"Items":[]}}}\n' + "kubectl.kubernetes.io/last-applied-configuration": '{"apiVersion":"codeflare.dev/v1beta2","kind":"AppWrapper","metadata":{"annotations":{},"name":"quicktest1","namespace":"ns"},"spec":{"components":["template":{"apiVersion":"ray.io/v1","kind":"RayCluster","metadata":{"labels":{"appwrapper.codeflare.dev":"quicktest1","controller-tools.k8s.io":"1.0"},"name":"quicktest1","namespace":"ns"},"spec":{"autoscalerOptions":{"idleTimeoutSeconds":60,"imagePullPolicy":"Always","resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"500m","memory":"512Mi"}},"upscalingMode":"Default"},"enableInTreeAutoscaling":false,"headGroupSpec":{"rayStartParams":{"block":"true","dashboard-host":"0.0.0.0","num-gpus":"0"},"serviceType":"ClusterIP","template":{"spec":{"containers":[{"image":"ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103","imagePullPolicy":"Always","lifecycle":{"preStop":{"exec":{"command":["/bin/sh","-c","ray stop"]}}},"name":"ray-head","ports":[{"containerPort":6379,"name":"gcs"},{"containerPort":8265,"name":"dashboard"},{"containerPort":10001,"name":"client"}],"resources":{"limits":{"cpu":2,"memory":"8G","nvidia.com/gpu":0},"requests":{"cpu":2,"memory":"8G","nvidia.com/gpu":0}}}]}}},"rayVersion":"1.12.0","workerGroupSpecs":[{"groupName":"small-group-quicktest","maxReplicas":1,"minReplicas":1,"rayStartParams":{"block":"true","num-gpus":"0"},"replicas":1,"template":{"metadata":{"annotations":{"key":"value"},"labels":{"key":"value"}},"spec":{"containers":[{"env":[{"name":"MY_POD_IP","valueFrom":{"fieldRef":{"fieldPath":"status.podIP"}}}],"image":"ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103","lifecycle":{"preStop":{"exec":{"command":["/bin/sh","-c","ray stop"]}}},"name":"machine-learning","resources":{"limits":{"cpu":1,"memory":"2G","nvidia.com/gpu":0},"requests":{"cpu":1,"memory":"2G","nvidia.com/gpu":0}}}],}}}]}},"replicas":1},{"template":{"apiVersion":"route.openshift.io/v1","kind":"Route","metadata":{"labels":{"odh-ray-cluster-service":"quicktest-head-svc"},"name":"ray-dashboard-quicktest","namespace":"default"},"spec":{"port":{"targetPort":"dashboard"},"to":{"kind":"Service","name":"quicktest-head-svc"}}}}]}\n' }, "creationTimestamp": "2023-02-22T16:26:07Z", "generation": 4, "managedFields": [ { - "apiVersion": "workload.codeflare.dev/v1beta1", + "apiVersion": "workload.codeflare.dev/v1beta2", "fieldsType": "FieldsV1", "fieldsV1": { "f:spec": { - "f:resources": { - "f:GenericItems": {}, - "f:metadata": {}, - }, - "f:schedulingSpec": {}, - "f:service": {".": {}, "f:spec": {}}, + "f:components": {}, + "f:suspend": {}, }, "f:status": { ".": {}, - "f:canrun": {}, "f:conditions": {}, - "f:controllerfirsttimestamp": {}, - "f:filterignore": {}, - "f:queuejobstate": {}, - "f:sender": {}, - "f:state": {}, - "f:systempriority": {}, + "f:phase": {}, }, }, "manager": "Go-http-client", @@ -1471,7 +1428,7 @@ def get_aw_obj(group, version, namespace, plural): "time": "2023-02-22T16:26:07Z", }, { - "apiVersion": "workload.codeflare.dev/v1beta1", + "apiVersion": "workload.codeflare.dev/v1beta2", "fieldsType": "FieldsV1", "fieldsV1": { "f:metadata": { @@ -1480,11 +1437,6 @@ def get_aw_obj(group, version, namespace, plural): "f:kubectl.kubernetes.io/last-applied-configuration": {}, } }, - "f:spec": { - ".": {}, - "f:priority": {}, - "f:resources": {".": {}, "f:Items": {}}, - }, }, "manager": "kubectl-client-side-apply", "operation": "Update", @@ -1497,83 +1449,135 @@ def get_aw_obj(group, version, namespace, plural): "uid": "6334fc1b-471e-4876-8e7b-0b2277679235", }, "spec": { - "priority": 9, - "resources": { - "GenericItems": [ - { - "allocated": 0, - "custompodresources": [ - { - "limits": { - "cpu": "2", - "memory": "8G", - "nvidia.com/gpu": "0", - }, - "replicas": 1, - "requests": { - "cpu": "2", - "memory": "8G", - "nvidia.com/gpu": "0", - }, + "components": [ + { + "podspecs": [ + { + "replicas": 1, + "path": "template.spec.headGroupSpec.template", + }, + { + "replicas": 3, + "path": "template.spec.workerGroupSpecs[0].template", + }, + ], + "template": { + "apiVersion": "ray.io/v1", + "kind": "RayCluster", + "metadata": { + "annotations": { + "sdk.codeflare.dev/local_interactive": "False" }, - { - "limits": { - "cpu": "1", - "memory": "2G", - "nvidia.com/gpu": "0", - }, - "replicas": 1, - "requests": { - "cpu": "1", - "memory": "2G", - "nvidia.com/gpu": "0", + "labels": { + "workload.codeflare.dev/appwrapper": "quicktest1", + "controller-tools.k8s.io": "1.0", + }, + "name": "quicktest1", + "namespace": "ns", + }, + "spec": { + "autoscalerOptions": { + "idleTimeoutSeconds": 60, + "imagePullPolicy": "Always", + "resources": { + "limits": { + "cpu": "500m", + "memory": "512Mi", + }, + "requests": { + "cpu": "500m", + "memory": "512Mi", + }, }, + "upscalingMode": "Default", }, - ], - "generictemplate": { - "apiVersion": "ray.io/v1", - "kind": "RayCluster", - "metadata": { - "annotations": { - "sdk.codeflare.dev/local_interactive": "False" + "enableInTreeAutoscaling": False, + "headGroupSpec": { + "rayStartParams": { + "block": "true", + "dashboard-host": "0.0.0.0", + "num-gpus": "0", }, - "labels": { - "workload.codeflare.dev/appwrapper": "quicktest1", - "controller-tools.k8s.io": "1.0", + "serviceType": "ClusterIP", + "template": { + "spec": { + "containers": [ + { + "image": "ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103", + "imagePullPolicy": "Always", + "lifecycle": { + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "ray stop", + ] + } + } + }, + "name": "ray-head", + "ports": [ + { + "containerPort": 6379, + "name": "gcs", + }, + { + "containerPort": 8265, + "name": "dashboard", + }, + { + "containerPort": 10001, + "name": "client", + }, + ], + "resources": { + "limits": { + "cpu": 2, + "memory": "8G", + "nvidia.com/gpu": 0, + }, + "requests": { + "cpu": 2, + "memory": "8G", + "nvidia.com/gpu": 0, + }, + }, + } + ] + } }, - "name": "quicktest1", - "namespace": "ns", }, - "spec": { - "autoscalerOptions": { - "idleTimeoutSeconds": 60, - "imagePullPolicy": "Always", - "resources": { - "limits": { - "cpu": "500m", - "memory": "512Mi", - }, - "requests": { - "cpu": "500m", - "memory": "512Mi", - }, - }, - "upscalingMode": "Default", - }, - "enableInTreeAutoscaling": False, - "headGroupSpec": { + "rayVersion": "1.12.0", + "workerGroupSpecs": [ + { + "groupName": "small-group-quicktest", + "maxReplicas": 1, + "minReplicas": 1, "rayStartParams": { "block": "true", - "dashboard-host": "0.0.0.0", "num-gpus": "0", }, - "serviceType": "ClusterIP", + "replicas": 1, "template": { + "metadata": { + "annotations": {"key": "value"}, + "labels": {"key": "value"}, + }, "spec": { "containers": [ { + "env": [ + { + "name": "MY_POD_IP", + "valueFrom": { + "fieldRef": { + "fieldPath": "status.podIP" + } + }, + } + ], "image": "ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103", - "imagePullPolicy": "Always", "lifecycle": { "preStop": { "exec": { @@ -1585,216 +1589,112 @@ def get_aw_obj(group, version, namespace, plural): } } }, - "name": "ray-head", - "ports": [ - { - "containerPort": 6379, - "name": "gcs", - }, - { - "containerPort": 8265, - "name": "dashboard", - }, - { - "containerPort": 10001, - "name": "client", - }, - ], + "name": "machine-learning", "resources": { "limits": { - "cpu": 2, - "memory": "8G", + "cpu": 1, + "memory": "2G", "nvidia.com/gpu": 0, }, "requests": { - "cpu": 2, - "memory": "8G", + "cpu": 1, + "memory": "2G", "nvidia.com/gpu": 0, }, }, } - ] - } - }, - }, - "rayVersion": "1.12.0", - "workerGroupSpecs": [ - { - "groupName": "small-group-quicktest", - "maxReplicas": 1, - "minReplicas": 1, - "rayStartParams": { - "block": "true", - "num-gpus": "0", - }, - "replicas": 1, - "template": { - "metadata": { - "annotations": {"key": "value"}, - "labels": {"key": "value"}, - }, - "spec": { - "containers": [ - { - "env": [ - { - "name": "MY_POD_IP", - "valueFrom": { - "fieldRef": { - "fieldPath": "status.podIP" - } - }, - } - ], - "image": "ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103", - "lifecycle": { - "preStop": { - "exec": { - "command": [ - "/bin/sh", - "-c", - "ray stop", - ] - } - } - }, - "name": "machine-learning", - "resources": { - "limits": { - "cpu": 1, - "memory": "2G", - "nvidia.com/gpu": 0, - }, - "requests": { - "cpu": 1, - "memory": "2G", - "nvidia.com/gpu": 0, - }, - }, - } - ], - }, + ], }, - } - ], - }, + }, + } + ], }, - "metadata": {}, - "priority": 0, - "priorityslope": 0, - "replicas": 1, }, - { - "allocated": 0, - "generictemplate": { - "apiVersion": "networking.k8s.io/v1", - "kind": "Ingress", - "metadata": { - "labels": { - "ingress-owner": "appwrapper-name", - "ingress-options": "false", - }, - "name": "ray-dashboard-quicktest", - "namespace": "default", + }, + { + "template": { + "apiVersion": "networking.k8s.io/v1", + "kind": "Ingress", + "metadata": { + "labels": { + "ingress-owner": "appwrapper-name", + "ingress-options": "false", }, - "spec": { - "ingressClassName": "nginx", - "rules": [ - { - "http": { - "paths": { - "backend": { - "service": { - "name": "quicktest-head-svc", - "port": { - "number": 8265 - }, - }, + "name": "ray-dashboard-quicktest", + "namespace": "default", + }, + "spec": { + "ingressClassName": "nginx", + "rules": [ + { + "http": { + "paths": { + "backend": { + "service": { + "name": "quicktest-head-svc", + "port": {"number": 8265}, }, - "pathType": "Prefix", - "path": "/", }, + "pathType": "Prefix", + "path": "/", }, - "host": "quicktest.awsroute.com", - } - ], - }, + }, + "host": "quicktest.awsroute.com", + } + ], }, - "metadata": {}, - "priority": 0, - "priorityslope": 0, }, - ], - "Items": [], - "metadata": {}, - }, - "schedulingSpec": {}, - "service": {"spec": {}}, + "metadata": {}, + }, + ], + "metadata": {}, }, "status": { - "canrun": True, "conditions": [ { "lastTransitionMicroTime": "2023-02-22T16:26:07.559447Z", "lastUpdateMicroTime": "2023-02-22T16:26:07.559447Z", "status": "True", - "type": "Init", + "type": "QuotaReserved", }, { - "lastTransitionMicroTime": "2023-02-22T16:26:07.559551Z", - "lastUpdateMicroTime": "2023-02-22T16:26:07.559551Z", - "reason": "AwaitingHeadOfLine", + "lastTransitionMicroTime": "2023-02-22T16:26:13.220564Z", + "lastUpdateMicroTime": "2023-02-22T16:26:13.220564Z", "status": "True", - "type": "Queueing", + "type": "ResourcesDeployed", }, { "lastTransitionMicroTime": "2023-02-22T16:26:13.220564Z", "lastUpdateMicroTime": "2023-02-22T16:26:13.220564Z", - "reason": "AppWrapperRunnable", "status": "True", - "type": "Dispatched", + "type": "PodsReady", }, ], - "controllerfirsttimestamp": "2023-02-22T16:26:07.559447Z", - "filterignore": True, - "queuejobstate": "Dispatched", - "sender": "before manageQueueJob - afterEtcdDispatching", - "state": "Running", - "systempriority": 9, + "phase": "Running", }, }, { - "apiVersion": "workload.codeflare.dev/v1beta1", + "apiVersion": "workload.codeflare.dev/v1beta2", "kind": "AppWrapper", "metadata": { "annotations": { - "kubectl.kubernetes.io/last-applied-configuration": '{"apiVersion":"codeflare.dev/v1beta1","kind":"AppWrapper","metadata":{"annotations":{},"name":"quicktest2","namespace":"ns"},"spec":{"priority":9,"resources":{"GenericItems":[{"custompodresources":[{"limits":{"cpu":2,"memory":"8G","nvidia.com/gpu":0},"replicas":1,"requests":{"cpu":2,"memory":"8G","nvidia.com/gpu":0}},{"limits":{"cpu":1,"memory":"2G","nvidia.com/gpu":0},"replicas":1,"requests":{"cpu":1,"memory":"2G","nvidia.com/gpu":0}}],"generictemplate":{"apiVersion":"ray.io/v1","kind":"RayCluster","metadata":{"labels":{"appwrapper.codeflare.dev":"quicktest2","controller-tools.k8s.io":"1.0"},"name":"quicktest2","namespace":"ns"},"spec":{"autoscalerOptions":{"idleTimeoutSeconds":60,"imagePullPolicy":"Always","resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"500m","memory":"512Mi"}},"upscalingMode":"Default"},"enableInTreeAutoscaling":false,"headGroupSpec":{"rayStartParams":{"block":"true","dashboard-host":"0.0.0.0","num-gpus":"0"},"serviceType":"ClusterIP","template":{"spec":{"containers":[{"image":"ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103","imagePullPolicy":"Always","lifecycle":{"preStop":{"exec":{"command":["/bin/sh","-c","ray stop"]}}},"name":"ray-head","ports":[{"containerPort":6379,"name":"gcs"},{"containerPort":8265,"name":"dashboard"},{"containerPort":10001,"name":"client"}],"resources":{"limits":{"cpu":2,"memory":"8G","nvidia.com/gpu":0},"requests":{"cpu":2,"memory":"8G","nvidia.com/gpu":0}}}]}}},"rayVersion":"1.12.0","workerGroupSpecs":[{"groupName":"small-group-quicktest","maxReplicas":1,"minReplicas":1,"rayStartParams":{"block":"true","num-gpus":"0"},"replicas":1,"template":{"metadata":{"annotations":{"key":"value"},"labels":{"key":"value"}},"spec":{"containers":[{"env":[{"name":"MY_POD_IP","valueFrom":{"fieldRef":{"fieldPath":"status.podIP"}}}],"image":"ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103","lifecycle":{"preStop":{"exec":{"command":["/bin/sh","-c","ray stop"]}}},"name":"machine-learning","resources":{"limits":{"cpu":1,"memory":"2G","nvidia.com/gpu":0},"requests":{"cpu":1,"memory":"2G","nvidia.com/gpu":0}}}],}}}]}},"replicas":1},{"generictemplate":{"apiVersion":"route.openshift.io/v1","kind":"Route","metadata":{"labels":{"odh-ray-cluster-service":"quicktest-head-svc"},"name":"ray-dashboard-quicktest","namespace":"default"},"spec":{"port":{"targetPort":"dashboard"},"to":{"kind":"Service","name":"quicktest-head-svc"}}},"replica":1}],"Items":[]}}}\n' + "kubectl.kubernetes.io/last-applied-configuration": '{"apiVersion":"codeflare.dev/v1beta2","kind":"AppWrapper","metadata":{"annotations":{},"name":"quicktest2","namespace":"ns"},"spec":{"components":[{"template":{"apiVersion":"ray.io/v1","kind":"RayCluster","metadata":{"labels":{"appwrapper.codeflare.dev":"quicktest2","controller-tools.k8s.io":"1.0"},"name":"quicktest2","namespace":"ns"},"spec":{"autoscalerOptions":{"idleTimeoutSeconds":60,"imagePullPolicy":"Always","resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"500m","memory":"512Mi"}},"upscalingMode":"Default"},"enableInTreeAutoscaling":false,"headGroupSpec":{"rayStartParams":{"block":"true","dashboard-host":"0.0.0.0","num-gpus":"0"},"serviceType":"ClusterIP","template":{"spec":{"containers":[{"image":"ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103","imagePullPolicy":"Always","lifecycle":{"preStop":{"exec":{"command":["/bin/sh","-c","ray stop"]}}},"name":"ray-head","ports":[{"containerPort":6379,"name":"gcs"},{"containerPort":8265,"name":"dashboard"},{"containerPort":10001,"name":"client"}],"resources":{"limits":{"cpu":2,"memory":"8G","nvidia.com/gpu":0},"requests":{"cpu":2,"memory":"8G","nvidia.com/gpu":0}}}]}}},"rayVersion":"1.12.0","workerGroupSpecs":[{"groupName":"small-group-quicktest","maxReplicas":1,"minReplicas":1,"rayStartParams":{"block":"true","num-gpus":"0"},"replicas":1,"template":{"metadata":{"annotations":{"key":"value"},"labels":{"key":"value"}},"spec":{"containers":[{"env":[{"name":"MY_POD_IP","valueFrom":{"fieldRef":{"fieldPath":"status.podIP"}}}],"image":"ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103","lifecycle":{"preStop":{"exec":{"command":["/bin/sh","-c","ray stop"]}}},"name":"machine-learning","resources":{"limits":{"cpu":1,"memory":"2G","nvidia.com/gpu":0},"requests":{"cpu":1,"memory":"2G","nvidia.com/gpu":0}}}],}}}]}},"replicas":1},{"generictemplate":{"apiVersion":"route.openshift.io/v1","kind":"Route","metadata":{"labels":{"odh-ray-cluster-service":"quicktest-head-svc"},"name":"ray-dashboard-quicktest","namespace":"default"},"spec":{"port":{"targetPort":"dashboard"},"to":{"kind":"Service","name":"quicktest-head-svc"}}},"replica":1}]}}\n' }, "creationTimestamp": "2023-02-22T16:26:07Z", "generation": 4, "managedFields": [ { - "apiVersion": "workload.codeflare.dev/v1beta1", + "apiVersion": "workload.codeflare.dev/v1beta2", "fieldsType": "FieldsV1", "fieldsV1": { "f:spec": { - "f:resources": { - "f:GenericItems": {}, - "f:metadata": {}, - }, - "f:schedulingSpec": {}, - "f:service": {".": {}, "f:spec": {}}, + "f:components": {}, + "f:suspend": {}, }, "f:status": { ".": {}, - "f:canrun": {}, "f:conditions": {}, - "f:controllerfirsttimestamp": {}, - "f:filterignore": {}, - "f:queuejobstate": {}, - "f:sender": {}, - "f:state": {}, - "f:systempriority": {}, + "f:phase": {}, }, }, "manager": "Go-http-client", @@ -1802,7 +1702,7 @@ def get_aw_obj(group, version, namespace, plural): "time": "2023-02-22T16:26:07Z", }, { - "apiVersion": "workload.codeflare.dev/v1beta1", + "apiVersion": "workload.codeflare.dev/v1beta2", "fieldsType": "FieldsV1", "fieldsV1": { "f:metadata": { @@ -1811,11 +1711,6 @@ def get_aw_obj(group, version, namespace, plural): "f:kubectl.kubernetes.io/last-applied-configuration": {}, } }, - "f:spec": { - ".": {}, - "f:priority": {}, - "f:resources": {".": {}, "f:Items": {}}, - }, }, "manager": "kubectl-client-side-apply", "operation": "Update", @@ -1828,83 +1723,125 @@ def get_aw_obj(group, version, namespace, plural): "uid": "6334fc1b-471e-4876-8e7b-0b2277679235", }, "spec": { - "priority": 9, - "resources": { - "GenericItems": [ - { - "allocated": 0, - "custompodresources": [ - { - "limits": { - "cpu": "2", - "memory": "8G", - "nvidia.com/gpu": "0", - }, - "replicas": 1, - "requests": { - "cpu": "2", - "memory": "8G", - "nvidia.com/gpu": "0", - }, + "components": [ + { + "template": { + "apiVersion": "ray.io/v1", + "kind": "RayCluster", + "metadata": { + "annotations": { + "sdk.codeflare.dev/local_interactive": "False" }, - { - "limits": { - "cpu": "1", - "memory": "2G", - "nvidia.com/gpu": "0", - }, - "replicas": 1, - "requests": { - "cpu": "1", - "memory": "2G", - "nvidia.com/gpu": "0", + "labels": { + "workload.codeflare.dev/appwrapper": "quicktest2", + "controller-tools.k8s.io": "1.0", + }, + "name": "quicktest2", + "namespace": "ns", + }, + "spec": { + "autoscalerOptions": { + "idleTimeoutSeconds": 60, + "imagePullPolicy": "Always", + "resources": { + "limits": { + "cpu": "500m", + "memory": "512Mi", + }, + "requests": { + "cpu": "500m", + "memory": "512Mi", + }, }, + "upscalingMode": "Default", }, - ], - "generictemplate": { - "apiVersion": "ray.io/v1", - "kind": "RayCluster", - "metadata": { - "annotations": { - "sdk.codeflare.dev/local_interactive": "False" + "enableInTreeAutoscaling": False, + "headGroupSpec": { + "rayStartParams": { + "block": "true", + "dashboard-host": "0.0.0.0", + "num-gpus": "0", }, - "labels": { - "workload.codeflare.dev/appwrapper": "quicktest2", - "controller-tools.k8s.io": "1.0", + "serviceType": "ClusterIP", + "template": { + "spec": { + "containers": [ + { + "image": "ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103", + "imagePullPolicy": "Always", + "lifecycle": { + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "ray stop", + ] + } + } + }, + "name": "ray-head", + "ports": [ + { + "containerPort": 6379, + "name": "gcs", + }, + { + "containerPort": 8265, + "name": "dashboard", + }, + { + "containerPort": 10001, + "name": "client", + }, + ], + "resources": { + "limits": { + "cpu": 2, + "memory": "8G", + "nvidia.com/gpu": 0, + }, + "requests": { + "cpu": 2, + "memory": "8G", + "nvidia.com/gpu": 0, + }, + }, + } + ] + } }, - "name": "quicktest2", - "namespace": "ns", }, - "spec": { - "autoscalerOptions": { - "idleTimeoutSeconds": 60, - "imagePullPolicy": "Always", - "resources": { - "limits": { - "cpu": "500m", - "memory": "512Mi", - }, - "requests": { - "cpu": "500m", - "memory": "512Mi", - }, - }, - "upscalingMode": "Default", - }, - "enableInTreeAutoscaling": False, - "headGroupSpec": { + "rayVersion": "1.12.0", + "workerGroupSpecs": [ + { + "groupName": "small-group-quicktest", + "maxReplicas": 1, + "minReplicas": 1, "rayStartParams": { "block": "true", - "dashboard-host": "0.0.0.0", "num-gpus": "0", }, - "serviceType": "ClusterIP", + "replicas": 1, "template": { + "metadata": { + "annotations": {"key": "value"}, + "labels": {"key": "value"}, + }, "spec": { "containers": [ { + "env": [ + { + "name": "MY_POD_IP", + "valueFrom": { + "fieldRef": { + "fieldPath": "status.podIP" + } + }, + } + ], "image": "ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103", - "imagePullPolicy": "Always", "lifecycle": { "preStop": { "exec": { @@ -1916,166 +1853,57 @@ def get_aw_obj(group, version, namespace, plural): } } }, - "name": "ray-head", - "ports": [ - { - "containerPort": 6379, - "name": "gcs", - }, - { - "containerPort": 8265, - "name": "dashboard", - }, - { - "containerPort": 10001, - "name": "client", - }, - ], + "name": "machine-learning", "resources": { "limits": { - "cpu": 2, - "memory": "8G", + "cpu": 1, + "memory": "2G", "nvidia.com/gpu": 0, }, "requests": { - "cpu": 2, - "memory": "8G", + "cpu": 1, + "memory": "2G", "nvidia.com/gpu": 0, }, }, } - ] - } - }, - }, - "rayVersion": "1.12.0", - "workerGroupSpecs": [ - { - "groupName": "small-group-quicktest", - "maxReplicas": 1, - "minReplicas": 1, - "rayStartParams": { - "block": "true", - "num-gpus": "0", + ], }, - "replicas": 1, - "template": { - "metadata": { - "annotations": {"key": "value"}, - "labels": {"key": "value"}, - }, - "spec": { - "containers": [ - { - "env": [ - { - "name": "MY_POD_IP", - "valueFrom": { - "fieldRef": { - "fieldPath": "status.podIP" - } - }, - } - ], - "image": "ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103", - "lifecycle": { - "preStop": { - "exec": { - "command": [ - "/bin/sh", - "-c", - "ray stop", - ] - } - } - }, - "name": "machine-learning", - "resources": { - "limits": { - "cpu": 1, - "memory": "2G", - "nvidia.com/gpu": 0, - }, - "requests": { - "cpu": 1, - "memory": "2G", - "nvidia.com/gpu": 0, - }, - }, - } - ], - }, - }, - } - ], - }, + }, + } + ], }, - "metadata": {}, - "priority": 0, - "priorityslope": 0, - "replicas": 1, }, - { - "allocated": 0, - "generictemplate": { - "apiVersion": "route.openshift.io/v1", - "kind": "Route", - "metadata": { - "labels": { - "odh-ray-cluster-service": "quicktest-head-svc" - }, - "name": "ray-dashboard-quicktest", - "namespace": "default", + "metadata": {}, + "priority": 0, + "priorityslope": 0, + "replicas": 1, + }, + { + "template": { + "apiVersion": "route.openshift.io/v1", + "kind": "Route", + "metadata": { + "labels": { + "odh-ray-cluster-service": "quicktest-head-svc" }, - "spec": { - "port": {"targetPort": "dashboard"}, - "to": { - "kind": "Service", - "name": "quicktest-head-svc", - }, + "name": "ray-dashboard-quicktest", + "namespace": "default", + }, + "spec": { + "port": {"targetPort": "dashboard"}, + "to": { + "kind": "Service", + "name": "quicktest-head-svc", }, }, - "metadata": {}, - "priority": 0, - "priorityslope": 0, }, - ], - "Items": [], - "metadata": {}, - }, - "schedulingSpec": {}, - "service": {"spec": {}}, - }, - "status": { - "canrun": True, - "conditions": [ - { - "lastTransitionMicroTime": "2023-02-22T16:26:07.559447Z", - "lastUpdateMicroTime": "2023-02-22T16:26:07.559447Z", - "status": "True", - "type": "Init", - }, - { - "lastTransitionMicroTime": "2023-02-22T16:26:07.559551Z", - "lastUpdateMicroTime": "2023-02-22T16:26:07.559551Z", - "reason": "AwaitingHeadOfLine", - "status": "True", - "type": "Queueing", - }, - { - "lastTransitionMicroTime": "2023-02-22T16:26:13.220564Z", - "lastUpdateMicroTime": "2023-02-22T16:26:13.220564Z", - "reason": "AppWrapperRunnable", - "status": "True", - "type": "Dispatched", }, ], - "controllerfirsttimestamp": "2023-02-22T16:26:07.559447Z", - "filterignore": True, - "queuejobstate": "Dispatched", - "sender": "before manageQueueJob - afterEtcdDispatching", - "state": "Pending", - "systempriority": 9, + }, + "status": { + "conditions": [], + "phase": "Suspended", }, }, ] @@ -2170,10 +1998,6 @@ def custom_side_effect(group, version, namespace, plural, **kwargs): cluster = get_cluster("quicktest") cluster_config = cluster.config assert cluster_config.name == "quicktest" and cluster_config.namespace == "ns" - assert ( - "m4.xlarge" in cluster_config.machine_types - and "g4dn.xlarge" in cluster_config.machine_types - ) assert cluster_config.min_cpus == 1 and cluster_config.max_cpus == 1 assert cluster_config.min_memory == 2 and cluster_config.max_memory == 2 assert cluster_config.num_gpus == 0 @@ -2203,15 +2027,9 @@ def test_get_cluster(mocker): cluster = get_cluster("quicktest") cluster_config = cluster.config assert cluster_config.name == "quicktest" and cluster_config.namespace == "ns" - assert ( - "m4.xlarge" in cluster_config.machine_types - and "g4dn.xlarge" in cluster_config.machine_types - ) assert cluster_config.min_cpus == 1 and cluster_config.max_cpus == 1 assert cluster_config.min_memory == 2 and cluster_config.max_memory == 2 assert cluster_config.num_gpus == 0 - assert cluster_config.instascale - assert cluster_config.local_interactive assert ( cluster_config.image == "ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103" @@ -2349,18 +2167,16 @@ def test_list_queue(mocker, capsys): list_all_queued("ns") captured = capsys.readouterr() assert captured.out == ( - "โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ\n" - "โ”‚ ๐Ÿš€ Cluster Queue Status โ”‚\n" - "โ”‚ ๐Ÿš€ โ”‚\n" - "โ”‚ +------------+---------+ โ”‚\n" - "โ”‚ | Name | Status | โ”‚\n" - "โ”‚ +============+=========+ โ”‚\n" - "โ”‚ | quicktest1 | running | โ”‚\n" - "โ”‚ | | | โ”‚\n" - "โ”‚ | quicktest2 | pending | โ”‚\n" - "โ”‚ | | | โ”‚\n" - "โ”‚ +------------+---------+ โ”‚\n" - "โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ\n" + "โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ\n" + "โ”‚ ๐Ÿš€ Cluster Queue Status โ”‚\n" + "โ”‚ ๐Ÿš€ โ”‚\n" + "โ”‚ +------------+-----------+ โ”‚\n" + "โ”‚ | Name | Status | โ”‚\n" + "โ”‚ +============+===========+ โ”‚\n" + "โ”‚ | quicktest2 | suspended | โ”‚\n" + "โ”‚ | | | โ”‚\n" + "โ”‚ +------------+-----------+ โ”‚\n" + "โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ\n" ) @@ -2368,7 +2184,8 @@ def test_cluster_status(mocker): mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch("kubernetes.config.load_kube_config", return_value="ignore") fake_aw = AppWrapper( - "test", AppWrapperStatus.FAILED, can_run=True, job_state="unused" + "test", + AppWrapperStatus.FAILED, ) fake_ray = RayCluster( name="test", @@ -2406,22 +2223,22 @@ def test_cluster_status(mocker): assert status == CodeFlareClusterStatus.FAILED assert ready == False - fake_aw.status = AppWrapperStatus.DELETED + fake_aw.status = AppWrapperStatus.FAILED status, ready = cf.status() assert status == CodeFlareClusterStatus.FAILED assert ready == False - fake_aw.status = AppWrapperStatus.PENDING + fake_aw.status = AppWrapperStatus.SUSPENDED status, ready = cf.status() assert status == CodeFlareClusterStatus.QUEUED assert ready == False - fake_aw.status = AppWrapperStatus.COMPLETED + fake_aw.status = AppWrapperStatus.RUNNING status, ready = cf.status() assert status == CodeFlareClusterStatus.STARTING assert ready == False - fake_aw.status = AppWrapperStatus.RUNNING_HOLD_COMPLETION + fake_aw.status = AppWrapperStatus.RUNNING status, ready = cf.status() assert status == CodeFlareClusterStatus.STARTING assert ready == False @@ -2613,6 +2430,7 @@ def test_DDPJobDefinition_dry_run_no_cluster(mocker): assert type(ddp_job._cfg) == type(dict()) assert type(ddp_job._scheduler) == type(str()) + # TODO: torchx still generates v1beta1 AppWrappers assert ( ddp_job.request.resource["spec"]["resources"]["GenericItems"][0][ "generictemplate" @@ -2899,7 +2717,7 @@ def test_AWManager_creation(mocker): def arg_check_aw_apply_effect(group, version, namespace, plural, body, *args): assert group == "workload.codeflare.dev" - assert version == "v1beta1" + assert version == "v1beta2" assert namespace == "ns" assert plural == "appwrappers" with open(f"{aw_dir}test.yaml") as f: @@ -2910,7 +2728,7 @@ def arg_check_aw_apply_effect(group, version, namespace, plural, body, *args): def arg_check_aw_del_effect(group, version, namespace, plural, name, *args): assert group == "workload.codeflare.dev" - assert version == "v1beta1" + assert version == "v1beta2" assert namespace == "ns" assert plural == "appwrappers" assert name == "test" @@ -3076,14 +2894,12 @@ def test_enable_local_interactive(mocker): {"name": "RAY_TLS_CA_CERT", "value": "/home/ray/workspace/tls/ca.crt"}, ] assert aw_spec != None - enable_local_interactive(aw_spec, cluster_name, namespace, ingress_domain) - head_group_spec = aw_spec["resources"]["GenericItems"][0]["generictemplate"][ - "spec" - ]["headGroupSpec"] - worker_group_spec = aw_spec["resources"]["GenericItems"][0]["generictemplate"][ - "spec" - ]["workerGroupSpecs"] - ca_secret = aw_spec["resources"]["GenericItems"][5]["generictemplate"] + enable_local_interactive( + aw_spec["components"], cluster_name, namespace, ingress_domain + ) + head_group_spec = aw_spec["components"][0]["template"]["spec"]["headGroupSpec"] + worker_group_spec = aw_spec["components"][0]["template"]["spec"]["workerGroupSpecs"] + ca_secret = aw_spec["components"][5]["template"] # At a minimal, make sure the following items are presented in the appwrapper spec.resources. # 1. headgroup has the initContainers command to generated TLS cert from the mounted CA cert. # Note: In this particular command, the DNS.5 in [alt_name] must match the exposed local_client_url: rayclient-{cluster_name}.{namespace}.{ingress_domain} @@ -3139,7 +2955,7 @@ def test_enable_local_interactive(mocker): assert ca_secret["metadata"]["namespace"] == namespace # 5. Rayclient ingress - Kind - rayclient_ingress = aw_spec["resources"]["GenericItems"][3]["generictemplate"] + rayclient_ingress = aw_spec["components"][3]["template"] paths = [ { "backend": { @@ -3267,9 +3083,9 @@ def test_gen_app_wrapper_with_oauth(mocker: MockerFixture): user_yaml = write_user_appwrapper.call_args.args[0] assert any( container["name"] == "oauth-proxy" - for container in user_yaml["spec"]["resources"]["GenericItems"][0][ - "generictemplate" - ]["spec"]["headGroupSpec"]["template"]["spec"]["containers"] + for container in user_yaml["spec"]["components"][0]["template"]["spec"][ + "headGroupSpec" + ]["template"]["spec"]["containers"] ) @@ -3431,10 +3247,9 @@ def test_rjc_list_jobs(ray_job_client, mocker): # Make sure to always keep this function last def test_cleanup(): os.remove(f"{aw_dir}unit-test-cluster.yaml") - os.remove(f"{aw_dir}prio-test-cluster.yaml") + os.remove(f"{aw_dir}unit-test-cluster-ray.yaml") os.remove(f"{aw_dir}test.yaml") os.remove(f"{aw_dir}raytest2.yaml") - os.remove(f"{aw_dir}unit-test-cluster-ray.yaml") os.remove("tls-cluster-namespace/ca.crt") os.remove("tls-cluster-namespace/tls.crt") os.remove("tls-cluster-namespace/tls.key") diff --git a/tests/unit_test_support.py b/tests/unit_test_support.py index c4b7416de..6e066155e 100644 --- a/tests/unit_test_support.py +++ b/tests/unit_test_support.py @@ -43,8 +43,6 @@ def createClusterConfig(): min_memory=5, max_memory=6, num_gpus=7, - instascale=True, - machine_types=["cpu.small", "gpu.large"], image_pull_secrets=["unit-test-pull-secret"], ingress_domain="apps.cluster.awsroute.org", image="quay.io/project-codeflare/ray:latest-py39-cu118", diff --git a/tests/upgrade/raycluster_sdk_upgrade_test.go b/tests/upgrade/raycluster_sdk_upgrade_test.go index 6b3c93cf0..1e008d644 100644 --- a/tests/upgrade/raycluster_sdk_upgrade_test.go +++ b/tests/upgrade/raycluster_sdk_upgrade_test.go @@ -21,8 +21,8 @@ import ( "testing" . "github.com/onsi/gomega" + mcadv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2" . "github.com/project-codeflare/codeflare-common/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" batchv1 "k8s.io/api/batch/v1" @@ -67,7 +67,7 @@ func TestMNISTRayClusterUp(t *testing.T) { policyRules := []rbacv1.PolicyRule{ { Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, - APIGroups: []string{mcadv1beta1.GroupName}, + APIGroups: []string{mcadv1beta2.GroupVersion.Group}, Resources: []string{"appwrappers"}, }, { @@ -244,7 +244,7 @@ func TestMnistJobSubmit(t *testing.T) { policyRules := []rbacv1.PolicyRule{ { Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, - APIGroups: []string{mcadv1beta1.GroupName}, + APIGroups: []string{mcadv1beta2.GroupVersion.Group}, Resources: []string{"appwrappers"}, }, {