Skip to content

Commit ed5b190

Browse files
bradmirobradmiro
committed
adding autoscaling samples
1 parent 39f5b4e commit ed5b190

File tree

2 files changed

+160
-0
lines changed

2 files changed

+160
-0
lines changed

‎dataproc/src/main/java/CreateClusterWithAutoscaling.java

+158
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
* This sample creates a Dataproc cluster with an autoscaling policy enabled. The policy we will be creating mirrors
17+
* the following YAML representation:
18+
*
19+
workerConfig:
20+
minInstances: 2
21+
maxInstances: 100
22+
weight: 1
23+
secondaryWorkerConfig:
24+
minInstances: 0
25+
maxInstances: 100
26+
weight: 1
27+
basicAlgorithm:
28+
cooldownPeriod: 4m
29+
yarnConfig:
30+
scaleUpFactor: 0.05
31+
scaleDownFactor: 1.0
32+
scaleUpMinWorkerFraction: 0.0
33+
scaleDownMinWorkerFraction: 0.0
34+
gracefulDecommissionTimeout: 1h
35+
*/
36+
37+
// [START dataproc_create_autoscaling_cluster]
38+
39+
import com.google.api.gax.longrunning.OperationFuture;
40+
import com.google.cloud.dataproc.v1.*;
41+
import com.google.protobuf.Duration;
42+
43+
import java.io.IOException;
44+
import java.util.concurrent.ExecutionException;
45+
46+
public class CreateClusterWithAutoscaling {
47+
48+
public static void createClusterwithAutoscaling() throws IOException, InterruptedException {
49+
// TODO(developer): Replace these variables before running the sample.
50+
String projectId = "your-project-id";
51+
String region = "your-project-region";
52+
String clusterName = "your-cluster-name";
53+
String autoscalingPolicyName = "your-autoscaling-policy";
54+
createClusterwithAutoscaling(projectId, region, clusterName, autoscalingPolicyName);
55+
}
56+
57+
public static void createClusterwithAutoscaling(
58+
String projectId, String region, String clusterName, String autoscalingPolicyName)
59+
throws IOException, InterruptedException {
60+
String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region);
61+
62+
// Configure the settings for the cluster controller client.
63+
ClusterControllerSettings clusterControllerSettings =
64+
ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build();
65+
66+
// Configure the settings for the autoscaling policy service client.
67+
AutoscalingPolicyServiceSettings autoscalingPolicyServiceSettings =
68+
AutoscalingPolicyServiceSettings.newBuilder().setEndpoint(myEndpoint).build();
69+
70+
// Create a cluster controller client and an autoscaling controller clien with the configured
71+
// settings. The clients only need to be created once and can be reused for multiple requests. Using a
72+
// try-with-resources closes the client, but this can also be done manually with the .close() method.
73+
try (ClusterControllerClient clusterControllerClient =
74+
ClusterControllerClient.create(clusterControllerSettings);
75+
AutoscalingPolicyServiceClient autoscalingPolicyServiceClient =
76+
AutoscalingPolicyServiceClient.create(autoscalingPolicyServiceSettings)) {
77+
78+
// Create the Autoscaling policy.
79+
InstanceGroupAutoscalingPolicyConfig workerInstanceGroupAutoscalingPolicyConfig =
80+
InstanceGroupAutoscalingPolicyConfig.newBuilder()
81+
.setMinInstances(2)
82+
.setMaxInstances(100)
83+
.setWeight(1)
84+
.build();
85+
InstanceGroupAutoscalingPolicyConfig secondaryWorkerInstanceGroupAutoscalingPolicyConfig =
86+
InstanceGroupAutoscalingPolicyConfig.newBuilder()
87+
.setMinInstances(0)
88+
.setMaxInstances(100)
89+
.setWeight(1)
90+
.build();
91+
BasicYarnAutoscalingConfig basicYarnApplicationConfig =
92+
BasicYarnAutoscalingConfig.newBuilder()
93+
.setScaleUpFactor(0.05)
94+
.setScaleDownFactor(1.0)
95+
.setScaleUpMinWorkerFraction(0.0)
96+
.setScaleUpMinWorkerFraction(0.0)
97+
.setGracefulDecommissionTimeout(Duration.newBuilder().setSeconds(3600).build())
98+
.build();
99+
BasicAutoscalingAlgorithm basicAutoscalingAlgorithm =
100+
BasicAutoscalingAlgorithm.newBuilder()
101+
.setCooldownPeriod(Duration.newBuilder().setSeconds(240).build())
102+
.setYarnConfig(basicYarnApplicationConfig)
103+
.build();
104+
AutoscalingPolicy autoscalingPolicy =
105+
AutoscalingPolicy.newBuilder()
106+
.setId(autoscalingPolicyName)
107+
.setWorkerConfig(workerInstanceGroupAutoscalingPolicyConfig)
108+
.setSecondaryWorkerConfig(secondaryWorkerInstanceGroupAutoscalingPolicyConfig)
109+
.setBasicAlgorithm(basicAutoscalingAlgorithm)
110+
.build();
111+
RegionName parent = RegionName.of(projectId, region);
112+
113+
// Policy is uploaded here.
114+
autoscalingPolicyServiceClient.createAutoscalingPolicy(parent, autoscalingPolicy);
115+
116+
// Now the policy can be referrenced when creating a cluster.
117+
String autoscalingPolicyURI =
118+
String.format(
119+
"projects/%s/locations/%s/autoscalingPolicies/%s", projectId, region, autoscalingPolicyName);
120+
AutoscalingConfig autoscalingConfig =
121+
AutoscalingConfig.newBuilder().setPolicyUri(autoscalingPolicyURI).build();
122+
123+
// Configure the settings for our cluster.
124+
InstanceGroupConfig masterConfig =
125+
InstanceGroupConfig.newBuilder()
126+
.setMachineTypeUri("n1-standard-1")
127+
.setNumInstances(1)
128+
.build();
129+
InstanceGroupConfig workerConfig =
130+
InstanceGroupConfig.newBuilder()
131+
.setMachineTypeUri("n1-standard-1")
132+
.setNumInstances(2)
133+
.build();
134+
ClusterConfig clusterConfig =
135+
ClusterConfig.newBuilder()
136+
.setMasterConfig(masterConfig)
137+
.setWorkerConfig(workerConfig)
138+
.setAutoscalingConfig(autoscalingConfig)
139+
.build();
140+
141+
// Create the cluster object with the desired cluster config.
142+
Cluster cluster =
143+
Cluster.newBuilder().setClusterName(clusterName).setConfig(clusterConfig).build();
144+
145+
// Create the Dataproc cluster.
146+
OperationFuture<Cluster, ClusterOperationMetadata> createClusterAsyncRequest =
147+
clusterControllerClient.createClusterAsync(projectId, region, cluster);
148+
Cluster response = createClusterAsyncRequest.get();
149+
150+
// Print out a success message.
151+
System.out.printf("Cluster created successfully: %s", response.getClusterName());
152+
153+
} catch (ExecutionException e) {
154+
System.err.println(String.format("Error executing createCluster: %s ", e.getMessage()));
155+
}
156+
}
157+
}
158+
// [END dataproc_create_autoscaling_cluster]

‎dataproc/src/test/java/CreateClusterWithAutoscalingTest.java

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
public class CreateClusterWithAutoscalingTest {
2+
}

0 commit comments

Comments
 (0)