Skip to content

Commit d1ac030

Browse files
bradmirobradmiro
authored
feat: adding CreateCluster sample for Dataproc (#1734)
* refactored and added tags to infinite speech streaming sample (#1605) * Changed 'main' region tag * Removed extra lines around tags and changed client import to v1 * Create dataproc directory and add CreateCluster sample * reverting changes to speech infinite streaming sample * Added java versions to pom * Several changes to file formatting as per request in the PR * Added comments to exceptions in CreateCluster, expanded exceptions and femoved endpoint configuring in CreateClusterTest.java * Fixed version for parent config * Added clarity to futures requests by expanding variables * Fixed linting errors * Fixed import ordering * Moved exceptions to function level in dataproc create cluster sample + test
1 parent 355721a commit d1ac030

File tree

3 files changed

+225
-0
lines changed

3 files changed

+225
-0
lines changed

‎dataproc/pom.xml

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<modelVersion>4.0.0</modelVersion>
6+
7+
<groupId>com.example</groupId>
8+
<artifactId>dataproc</artifactId>
9+
<version>1.0-SNAPSHOT</version>
10+
11+
<!--
12+
The parent pom defines common style checks and testing strategies for our samples.
13+
Removing or replacing it should not affect the execution of the samples in anyway.
14+
-->
15+
<parent>
16+
<groupId>com.google.cloud.samples</groupId>
17+
<artifactId>shared-configuration</artifactId>
18+
<version>1.0.11</version>
19+
</parent>
20+
21+
<properties>
22+
<maven.compiler.target>1.8</maven.compiler.target>
23+
<maven.compiler.source>1.8</maven.compiler.source>
24+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
25+
</properties>
26+
27+
<build>
28+
<pluginManagement>
29+
<plugins>
30+
<plugin>
31+
<groupId>org.apache.maven.plugins</groupId>
32+
<artifactId>maven-compiler-plugin</artifactId>
33+
<version>3.8.1</version>
34+
</plugin>
35+
</plugins>
36+
</pluginManagement>
37+
</build>
38+
39+
<dependencies>
40+
<!-- Test dependencies -->
41+
<dependency>
42+
<groupId>junit</groupId>
43+
<artifactId>junit</artifactId>
44+
<version>4.12</version>
45+
</dependency>
46+
<dependency>
47+
<groupId>com.google.cloud</groupId>
48+
<artifactId>google-cloud-dataproc</artifactId>
49+
<version>0.117.0</version>
50+
</dependency>
51+
</dependencies>
52+
53+
</project>

‎dataproc/src/main/java/CreateCluster.java

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
/*
2+
* Copyright 2019 Google Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
// [START create_cluster]
18+
import com.google.api.gax.longrunning.OperationFuture;
19+
import com.google.cloud.dataproc.v1.Cluster;
20+
import com.google.cloud.dataproc.v1.ClusterConfig;
21+
import com.google.cloud.dataproc.v1.ClusterControllerClient;
22+
import com.google.cloud.dataproc.v1.ClusterControllerSettings;
23+
import com.google.cloud.dataproc.v1.ClusterOperationMetadata;
24+
import com.google.cloud.dataproc.v1.InstanceGroupConfig;
25+
import java.io.IOException;
26+
import java.io.InterruptedIOException;
27+
import java.util.concurrent.ExecutionException;
28+
29+
public class CreateCluster {
30+
31+
public static void createCluster(String projectId, String region, String clusterName)
32+
throws IOException, InterruptedException {
33+
String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region);
34+
35+
// Configure the settings for the cluster controller client
36+
ClusterControllerSettings clusterControllerSettings =
37+
ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build();
38+
39+
// Create a cluster controller client with the configured settings. We only need to create
40+
// the client once, and can be reused for multiple requests. Using a try-with-resources
41+
// will close the client for us, but this can also be done manually with the .close() method.
42+
try (ClusterControllerClient clusterControllerClient = ClusterControllerClient
43+
.create(clusterControllerSettings)) {
44+
// Configure the settings for our cluster
45+
InstanceGroupConfig masterConfig = InstanceGroupConfig.newBuilder()
46+
.setMachineTypeUri("n1-standard-1")
47+
.setNumInstances(1)
48+
.build();
49+
InstanceGroupConfig workerConfig = InstanceGroupConfig.newBuilder()
50+
.setMachineTypeUri("n1-standard-1")
51+
.setNumInstances(2)
52+
.build();
53+
ClusterConfig clusterConfig = ClusterConfig.newBuilder()
54+
.setMasterConfig(masterConfig)
55+
.setWorkerConfig(workerConfig)
56+
.build();
57+
// Create the cluster object with the desired cluster config
58+
Cluster cluster = Cluster.newBuilder()
59+
.setClusterName(clusterName)
60+
.setConfig(clusterConfig)
61+
.build();
62+
63+
// Send a request to create a Dataproc cluster.
64+
OperationFuture<Cluster, ClusterOperationMetadata> createClusterAsyncRequest =
65+
clusterControllerClient.createClusterAsync(projectId, region, cluster);
66+
Cluster response = createClusterAsyncRequest.get();
67+
68+
// Print out the response
69+
System.out.println(
70+
String.format("Cluster created successfully: %s", response.getClusterName())
71+
);
72+
73+
} catch (IOException e) {
74+
// Likely this would occur due to issues authenticating with GCP. Make sure the environment
75+
// variable GOOGLE_APPLICATION_CREDENTIALS is configured.
76+
System.out.println("Error creating the cluster controller client: \n" + e.toString());
77+
} catch (ExecutionException e) {
78+
// Common issues for this include needing to increase compute engine quotas or a cluster of
79+
// the same name already exists.
80+
System.out.println("Error during cluster creation request: \n" + e.toString());
81+
}
82+
}
83+
}
84+
// [END create_cluster]

‎dataproc/src/test/java/CreateClusterTest.java

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/*
2+
* Copyright 2019 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
import static junit.framework.TestCase.assertNotNull;
18+
import static org.hamcrest.MatcherAssert.assertThat;
19+
20+
import com.google.api.gax.longrunning.OperationFuture;
21+
import com.google.cloud.dataproc.v1.ClusterControllerClient;
22+
import com.google.cloud.dataproc.v1.ClusterOperationMetadata;
23+
import com.google.protobuf.Empty;
24+
import java.io.ByteArrayOutputStream;
25+
import java.io.IOException;
26+
import java.io.PrintStream;
27+
import java.util.UUID;
28+
import java.util.concurrent.ExecutionException;
29+
import org.hamcrest.CoreMatchers;
30+
import org.junit.After;
31+
import org.junit.Before;
32+
import org.junit.BeforeClass;
33+
import org.junit.Test;
34+
import org.junit.runner.RunWith;
35+
import org.junit.runners.JUnit4;
36+
37+
@RunWith(JUnit4.class)
38+
public class CreateClusterTest {
39+
40+
private static final String BASE_CLUSTER_NAME = "test-cluster";
41+
private static final String REGION = "us-central1";
42+
43+
private static String projectId = System.getenv("GOOGLE_CLOUD_PROJECT");
44+
private String clusterName;
45+
private ByteArrayOutputStream bout;
46+
47+
private static void requireEnv(String varName) {
48+
assertNotNull(
49+
System.getenv(varName),
50+
String.format("Environment variable '%s' is required to perform these tests.", varName)
51+
);
52+
}
53+
54+
@BeforeClass
55+
public static void checkRequirements() {
56+
requireEnv("GOOGLE_APPLICATION_CREDENTIALS");
57+
requireEnv("GOOGLE_CLOUD_PROJECT");
58+
}
59+
60+
@Before
61+
public void setUp() {
62+
clusterName = String.format("%s-%s", BASE_CLUSTER_NAME, UUID.randomUUID().toString());
63+
64+
bout = new ByteArrayOutputStream();
65+
System.setOut(new PrintStream(bout));
66+
}
67+
68+
@Test
69+
public void createClusterTest() throws IOException, InterruptedException {
70+
CreateCluster.createCluster(projectId, REGION, clusterName);
71+
String output = bout.toString();
72+
73+
assertThat(output, CoreMatchers.containsString(clusterName));
74+
}
75+
76+
@After
77+
public void tearDown() throws IOException, InterruptedException {
78+
try (ClusterControllerClient clusterControllerClient = ClusterControllerClient
79+
.create()) {
80+
OperationFuture<Empty, ClusterOperationMetadata> deleteClusterAsyncRequest =
81+
clusterControllerClient.deleteClusterAsync(projectId, REGION, clusterName);
82+
deleteClusterAsyncRequest.get();
83+
84+
} catch (ExecutionException e) {
85+
System.out.println("Error during cluster deletion: \n" + e.toString());
86+
}
87+
}
88+
}

0 commit comments

Comments
 (0)