|
| 1 | +/* |
| 2 | +* Copyright 2020 Google LLC |
| 3 | +* |
| 4 | +* Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | +* you may not use this file except in compliance with the License. |
| 6 | +* You may obtain a copy of the License at |
| 7 | +* |
| 8 | +* http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | +* |
| 10 | +* Unless required by applicable law or agreed to in writing, software |
| 11 | +* distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | +* See the License for the specific language governing permissions and |
| 14 | +* limitations under the License. |
| 15 | +* |
| 16 | +* This sample creates a Dataproc cluster with an autoscaling policy enabled. |
| 17 | +* The policy we will be creating mirrors the following YAML representation: |
| 18 | +* |
| 19 | + workerConfig: |
| 20 | + minInstances: 2 |
| 21 | + maxInstances: 100 |
| 22 | + weight: 1 |
| 23 | + secondaryWorkerConfig: |
| 24 | + minInstances: 0 |
| 25 | + maxInstances: 100 |
| 26 | + weight: 1 |
| 27 | + basicAlgorithm: |
| 28 | + cooldownPeriod: 4m |
| 29 | + yarnConfig: |
| 30 | + scaleUpFactor: 0.05 |
| 31 | + scaleDownFactor: 1.0 |
| 32 | + scaleUpMinWorkerFraction: 0.0 |
| 33 | + scaleDownMinWorkerFraction: 0.0 |
| 34 | + gracefulDecommissionTimeout: 1h |
| 35 | +*/ |
| 36 | + |
| 37 | +// [START dataproc_create_autoscaling_cluster] |
| 38 | + |
| 39 | +import com.google.api.gax.longrunning.OperationFuture; |
| 40 | +import com.google.cloud.dataproc.v1.AutoscalingConfig; |
| 41 | +import com.google.cloud.dataproc.v1.AutoscalingPolicy; |
| 42 | +import com.google.cloud.dataproc.v1.AutoscalingPolicyServiceClient; |
| 43 | +import com.google.cloud.dataproc.v1.AutoscalingPolicyServiceSettings; |
| 44 | +import com.google.cloud.dataproc.v1.BasicAutoscalingAlgorithm; |
| 45 | +import com.google.cloud.dataproc.v1.BasicYarnAutoscalingConfig; |
| 46 | +import com.google.cloud.dataproc.v1.Cluster; |
| 47 | +import com.google.cloud.dataproc.v1.ClusterConfig; |
| 48 | +import com.google.cloud.dataproc.v1.ClusterControllerClient; |
| 49 | +import com.google.cloud.dataproc.v1.ClusterControllerSettings; |
| 50 | +import com.google.cloud.dataproc.v1.ClusterOperationMetadata; |
| 51 | +import com.google.cloud.dataproc.v1.InstanceGroupAutoscalingPolicyConfig; |
| 52 | +import com.google.cloud.dataproc.v1.InstanceGroupConfig; |
| 53 | +import com.google.cloud.dataproc.v1.RegionName; |
| 54 | +import com.google.protobuf.Duration; |
| 55 | +import java.io.IOException; |
| 56 | +import java.util.concurrent.ExecutionException; |
| 57 | + |
| 58 | +public class CreateClusterWithAutoscaling { |
| 59 | + |
| 60 | + public static void createClusterwithAutoscaling() throws IOException, InterruptedException { |
| 61 | + // TODO(developer): Replace these variables before running the sample. |
| 62 | + String projectId = "your-project-id"; |
| 63 | + String region = "your-project-region"; |
| 64 | + String clusterName = "your-cluster-name"; |
| 65 | + String autoscalingPolicyName = "your-autoscaling-policy"; |
| 66 | + createClusterwithAutoscaling(projectId, region, clusterName, autoscalingPolicyName); |
| 67 | + } |
| 68 | + |
| 69 | + public static void createClusterwithAutoscaling( |
| 70 | + String projectId, String region, String clusterName, String autoscalingPolicyName) |
| 71 | + throws IOException, InterruptedException { |
| 72 | + String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region); |
| 73 | + |
| 74 | + // Configure the settings for the cluster controller client. |
| 75 | + ClusterControllerSettings clusterControllerSettings = |
| 76 | + ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build(); |
| 77 | + |
| 78 | + // Configure the settings for the autoscaling policy service client. |
| 79 | + AutoscalingPolicyServiceSettings autoscalingPolicyServiceSettings = |
| 80 | + AutoscalingPolicyServiceSettings.newBuilder().setEndpoint(myEndpoint).build(); |
| 81 | + |
| 82 | + // Create a cluster controller client and an autoscaling controller client with the configured |
| 83 | + // settings. The clients only need to be created once and can be reused for multiple requests. |
| 84 | + // Using a |
| 85 | + // try-with-resources closes the client, but this can also be done manually with the .close() |
| 86 | + // method. |
| 87 | + try (ClusterControllerClient clusterControllerClient = |
| 88 | + ClusterControllerClient.create(clusterControllerSettings); |
| 89 | + AutoscalingPolicyServiceClient autoscalingPolicyServiceClient = |
| 90 | + AutoscalingPolicyServiceClient.create(autoscalingPolicyServiceSettings)) { |
| 91 | + |
| 92 | + // Create the Autoscaling policy. |
| 93 | + InstanceGroupAutoscalingPolicyConfig workerInstanceGroupAutoscalingPolicyConfig = |
| 94 | + InstanceGroupAutoscalingPolicyConfig.newBuilder() |
| 95 | + .setMinInstances(2) |
| 96 | + .setMaxInstances(100) |
| 97 | + .setWeight(1) |
| 98 | + .build(); |
| 99 | + InstanceGroupAutoscalingPolicyConfig secondaryWorkerInstanceGroupAutoscalingPolicyConfig = |
| 100 | + InstanceGroupAutoscalingPolicyConfig.newBuilder() |
| 101 | + .setMinInstances(0) |
| 102 | + .setMaxInstances(100) |
| 103 | + .setWeight(1) |
| 104 | + .build(); |
| 105 | + BasicYarnAutoscalingConfig basicYarnApplicationConfig = |
| 106 | + BasicYarnAutoscalingConfig.newBuilder() |
| 107 | + .setScaleUpFactor(0.05) |
| 108 | + .setScaleDownFactor(1.0) |
| 109 | + .setScaleUpMinWorkerFraction(0.0) |
| 110 | + .setScaleUpMinWorkerFraction(0.0) |
| 111 | + .setGracefulDecommissionTimeout(Duration.newBuilder().setSeconds(3600).build()) |
| 112 | + .build(); |
| 113 | + BasicAutoscalingAlgorithm basicAutoscalingAlgorithm = |
| 114 | + BasicAutoscalingAlgorithm.newBuilder() |
| 115 | + .setCooldownPeriod(Duration.newBuilder().setSeconds(240).build()) |
| 116 | + .setYarnConfig(basicYarnApplicationConfig) |
| 117 | + .build(); |
| 118 | + AutoscalingPolicy autoscalingPolicy = |
| 119 | + AutoscalingPolicy.newBuilder() |
| 120 | + .setId(autoscalingPolicyName) |
| 121 | + .setWorkerConfig(workerInstanceGroupAutoscalingPolicyConfig) |
| 122 | + .setSecondaryWorkerConfig(secondaryWorkerInstanceGroupAutoscalingPolicyConfig) |
| 123 | + .setBasicAlgorithm(basicAutoscalingAlgorithm) |
| 124 | + .build(); |
| 125 | + RegionName parent = RegionName.of(projectId, region); |
| 126 | + |
| 127 | + // Policy is uploaded here. |
| 128 | + autoscalingPolicyServiceClient.createAutoscalingPolicy(parent, autoscalingPolicy); |
| 129 | + |
| 130 | + // Now the policy can be referenced when creating a cluster. |
| 131 | + String autoscalingPolicyUri = |
| 132 | + String.format( |
| 133 | + "projects/%s/locations/%s/autoscalingPolicies/%s", |
| 134 | + projectId, region, autoscalingPolicyName); |
| 135 | + AutoscalingConfig autoscalingConfig = |
| 136 | + AutoscalingConfig.newBuilder().setPolicyUri(autoscalingPolicyUri).build(); |
| 137 | + |
| 138 | + // Configure the settings for our cluster. |
| 139 | + InstanceGroupConfig masterConfig = |
| 140 | + InstanceGroupConfig.newBuilder() |
| 141 | + .setMachineTypeUri("n1-standard-2") |
| 142 | + .setNumInstances(1) |
| 143 | + .build(); |
| 144 | + InstanceGroupConfig workerConfig = |
| 145 | + InstanceGroupConfig.newBuilder() |
| 146 | + .setMachineTypeUri("n1-standard-2") |
| 147 | + .setNumInstances(2) |
| 148 | + .build(); |
| 149 | + ClusterConfig clusterConfig = |
| 150 | + ClusterConfig.newBuilder() |
| 151 | + .setMasterConfig(masterConfig) |
| 152 | + .setWorkerConfig(workerConfig) |
| 153 | + .setAutoscalingConfig(autoscalingConfig) |
| 154 | + .build(); |
| 155 | + |
| 156 | + // Create the cluster object with the desired cluster config. |
| 157 | + Cluster cluster = |
| 158 | + Cluster.newBuilder().setClusterName(clusterName).setConfig(clusterConfig).build(); |
| 159 | + |
| 160 | + // Create the Dataproc cluster. |
| 161 | + OperationFuture<Cluster, ClusterOperationMetadata> createClusterAsyncRequest = |
| 162 | + clusterControllerClient.createClusterAsync(projectId, region, cluster); |
| 163 | + Cluster response = createClusterAsyncRequest.get(); |
| 164 | + |
| 165 | + // Print out a success message. |
| 166 | + System.out.printf("Cluster created successfully: %s", response.getClusterName()); |
| 167 | + |
| 168 | + } catch (ExecutionException e) { |
| 169 | + // If cluster creation does not complete successfully, print the error message. |
| 170 | + System.err.println(String.format("createClusterWithAutoscaling: %s ", e.getMessage())); |
| 171 | + } |
| 172 | + } |
| 173 | +} |
| 174 | +// [END dataproc_create_autoscaling_cluster] |
0 commit comments