docs(samples): Adding samples for template usage (#41)

m-strzelczyk · web-flow · commit 278d7e192c62 · 2022-09-30T13:50:14.000Z
Preparing another code sample. This time for the [`Create a job from a Compute Engine instance template`](https://cloud.google.com/batch/docs/create-run-job#create-job-instance-template) section.
diff --git a/compute/batch/requirements-test.txt b/compute/batch/requirements-test.txt
@@ -1,3 +1,4 @@
 pytest==7.1.3
-pytest-parallel==0.1.1
-google-cloud-storage==2.5.0
+google-cloud-compute==1.5.2
+google-cloud-resource-manager==1.6.1
+google-cloud-storage==2.5.0
diff --git a/compute/batch/snippets/create/create_with_template.py b/compute/batch/snippets/create/create_with_template.py
@@ -0,0 +1,87 @@
+#  Copyright 2022 Google LLC
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+# [START batch_create_job_with_template]
+from google.cloud import batch_v1
+
+
+def create_script_job_with_template(project_id: str, region: str, job_name: str, template_link: str) -> batch_v1.Job:
+    """
+    This method shows how to create a sample Batch Job that will run
+    a simple command on Cloud Compute instances created using a provided Template.
+
+    Args:
+        project_id: project ID or project number of the Cloud project you want to use.
+        region: name of the region you want to use to run the job. Regions that are
+            available for Batch are listed on: https://cloud.google.com/batch/docs/get-started#locations
+        job_name: the name of the job that will be created.
+            It needs to be unique for each project and region pair.
+        template_link: a link to an existing Instance Template. Acceptable formats:
+            * "projects/{project_id}/global/instanceTemplates/{template_name}"
+            * "{template_name}" - if the template is defined in the same project as used to create the Job.
+
+    Returns:
+        A job object representing the job created.
+    """
+    client = batch_v1.BatchServiceClient()
+
+    # Define what will be done as part of the job.
+    task = batch_v1.TaskSpec()
+    runnable = batch_v1.Runnable()
+    runnable.script = batch_v1.Runnable.Script()
+    runnable.script.text = "echo Hello world! This is task ${BATCH_TASK_INDEX}. This job has a total of ${BATCH_TASK_COUNT} tasks."
+    # You can also run a script from a file. Just remember, that needs to be a script that's
+    # already on the VM that will be running the job. Using runnable.script.text and runnable.script.path is mutually
+    # exclusive.
+    # runnable.script.path = '/tmp/test.sh'
+    task.runnables = [runnable]
+
+    # We can specify what resources are requested by each task.
+    resources = batch_v1.ComputeResource()
+    resources.cpu_milli = 2000  # in milliseconds per cpu-second. This means the task requires 2 whole CPUs.
+    resources.memory_mib = 16
+    task.compute_resource = resources
+
+    task.max_retry_count = 2
+    task.max_run_duration = "3600s"
+
+    # Tasks are grouped inside a job using TaskGroups.
+    group = batch_v1.TaskGroup()
+    group.task_count = 4
+    group.task_spec = task
+
+    # Policies are used to define on what kind of virtual machines the tasks will run on.
+    # In this case, we tell the system to use an instance template that defines all the
+    # required parameters.
+    allocation_policy = batch_v1.AllocationPolicy()
+    instances = batch_v1.AllocationPolicy.InstancePolicyOrTemplate()
+    instances.instance_template = template_link
+    allocation_policy.instances = [instances]
+
+    job = batch_v1.Job()
+    job.task_groups = [group]
+    job.allocation_policy = allocation_policy
+    job.labels = {"env": "testing", "type": "script"}
+    # We use Cloud Logging as it's an out of the box available option
+    job.logs_policy = batch_v1.LogsPolicy()
+    job.logs_policy.destination = batch_v1.LogsPolicy.Destination.CLOUD_LOGGING
+
+    create_request = batch_v1.CreateJobRequest()
+    create_request.job = job
+    create_request.job_id = job_name
+    # The job's parent is the region in which the job will run
+    create_request.parent = f"projects/{project_id}/locations/{region}"
+
+    return client.create_job(create_request)
+# [END batch_create_job_with_template]
diff --git a/compute/batch/snippets/tests/test_template.py b/compute/batch/snippets/tests/test_template.py
@@ -0,0 +1,114 @@
+#  Copyright 2022 Google LLC
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import uuid
+
+import google.auth
+from google.cloud import batch_v1
+from google.cloud import compute_v1
+from google.cloud import resourcemanager_v3
+import pytest
+
+
+from .test_basics import _test_body
+
+from ..create.create_with_template import create_script_job_with_template
+
+PROJECT = google.auth.default()[1]
+
+PROJECT_NUMBER = resourcemanager_v3.ProjectsClient().get_project(name=f"projects/{PROJECT}").name.split("/")[1]
+
+REGION = 'europe-north1'
+
+TIMEOUT = 600  # 10 minutes
+
+WAIT_STATES = {
+    batch_v1.JobStatus.State.STATE_UNSPECIFIED,
+    batch_v1.JobStatus.State.QUEUED,
+    batch_v1.JobStatus.State.RUNNING,
+    batch_v1.JobStatus.State.SCHEDULED,
+}
+
+
+@pytest.fixture
+def job_name():
+    return f"test-job-{uuid.uuid4().hex[:10]}"
+
+
+@pytest.fixture
+def instance_template():
+    disk = compute_v1.AttachedDisk()
+    initialize_params = compute_v1.AttachedDiskInitializeParams()
+    initialize_params.source_image = (
+        "projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts"
+    )
+    initialize_params.disk_size_gb = 25
+    initialize_params.disk_type = 'pd-balanced'
+    disk.initialize_params = initialize_params
+    disk.auto_delete = True
+    disk.boot = True
+
+    network_interface = compute_v1.NetworkInterface()
+    network_interface.name = "global/networks/default"
+
+    access = compute_v1.AccessConfig()
+    access.type_ = compute_v1.AccessConfig.Type.ONE_TO_ONE_NAT.name
+    access.name = "External NAT"
+    access.network_tier = access.NetworkTier.PREMIUM.name
+    network_interface.access_configs = [access]
+
+    template = compute_v1.InstanceTemplate()
+    template.name = "test-template-" + uuid.uuid4().hex[:10]
+    template.properties = compute_v1.InstanceProperties()
+    template.properties.disks = [disk]
+    template.properties.machine_type = "e2-standard-16"
+    template.properties.network_interfaces = [network_interface]
+
+    template.properties.scheduling = compute_v1.Scheduling()
+    template.properties.scheduling.on_host_maintenance = compute_v1.Scheduling.OnHostMaintenance.MIGRATE.name
+    template.properties.scheduling.provisioning_model = compute_v1.Scheduling.ProvisioningModel.STANDARD.name
+    template.properties.scheduling.automatic_restart = True
+
+    template.properties.service_accounts = [
+        {
+            "email": f"{PROJECT_NUMBER}-compute@developer.gserviceaccount.com",
+            "scopes": [
+                "https://www.googleapis.com/auth/devstorage.read_only",
+                "https://www.googleapis.com/auth/logging.write",
+                "https://www.googleapis.com/auth/monitoring.write",
+                "https://www.googleapis.com/auth/servicecontrol",
+                "https://www.googleapis.com/auth/service.management.readonly",
+                "https://www.googleapis.com/auth/trace.append"
+            ]
+        }
+    ]
+
+    template_client = compute_v1.InstanceTemplatesClient()
+    operation_client = compute_v1.GlobalOperationsClient()
+    op = template_client.insert_unary(
+        project=PROJECT, instance_template_resource=template
+    )
+    operation_client.wait(project=PROJECT, operation=op.name)
+
+    template = template_client.get(project=PROJECT, instance_template=template.name)
+
+    yield template
+
+    op = template_client.delete_unary(project=PROJECT, instance_template=template.name)
+    operation_client.wait(project=PROJECT, operation=op.name)
+
+
+def test_template_job(job_name, instance_template):
+    job = create_script_job_with_template(PROJECT, REGION, job_name, instance_template.self_link)
+    _test_body(job)
