feat: add spark history server api

Add api methods for managing spark history server.
Add submissionTime and autoTerminationTime fields to Application response models.

BREAKING CHANGE: Type of timestamp fields in Application response models has been changed to `Date`.

The type of timestamp fields like startTime, endTime in `ApplicationGetStateResponse` and `Application` models has been changed from `String` to `Date`.

Signed-off-by: Subin Shekhar <[email protected]>

Author: subinpc

modules/examples/src/main/java/com/ibm/cloud/iaesdk/ibm_analytics_engine_api/v3/IbmAnalyticsEngineApiExamples.java

@@ -32,6 +32,7 @@
 import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.GetLogForwardingConfigOptions;
 import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.GetLoggingConfigurationOptions;
 import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.GetResourceConsumptionLimitsOptions;
+import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.GetSparkHistoryServerOptions;
 import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.Instance;
 import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.InstanceGetStateResponse;
 import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.InstanceHomeResponse;
@@ -44,6 +45,9 @@
 import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.ResourceConsumptionLimitsResponse;
 import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.Runtime;
 import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.SetInstanceHomeOptions;
+import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.SparkHistoryServerResponse;
+import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.StartSparkHistoryServerOptions;
+import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.StopSparkHistoryServerOptions;
 import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.UpdateInstanceDefaultConfigsOptions;
 import com.ibm.cloud.sdk.core.http.Response;
 import com.ibm.cloud.sdk.core.service.exception.ServiceResponseException;
@@ -241,9 +245,9 @@ public static void main(String[] args) throws Exception {
       CreateApplicationOptions createApplicationOptions = new CreateApplicationOptions.Builder()
         .instanceId("dc0e9889-eab2-4t9e-9441-566209499546")
         .applicationDetails(new ApplicationRequestApplicationDetails.Builder()
-          .application("/opt/ibm/spark/examples/src/main/python/wordcount.py")
+        .application("/opt/ibm/spark/examples/src/main/python/wordcount.py")
           .addArguments("/opt/ibm/spark/examples/src/main/resources/people.txt")
-          .runtime(runtimeModel)
+        .runtime(runtimeModel)
           .build())
         .build();
 
@@ -263,8 +267,6 @@ public static void main(String[] args) throws Exception {
       ListApplicationsOptions listApplicationsOptions = new ListApplicationsOptions.Builder()
         .instanceId("dc0e9889-eab2-4t9e-9441-566209499546")
         .addState("accepted")
-        .addState("submitted")
-        .addState("waiting")
         .addState("running")
         .addState("finished")
         .addState("failed")
@@ -420,6 +422,54 @@ public static void main(String[] args) throws Exception {
           e.getStatusCode(), e.getMessage(), e.getDebuggingInfo()), e);
     }
 
+    try {
+      System.out.println("startSparkHistoryServer() result:");
+      // begin-start_spark_history_server
+      StartSparkHistoryServerOptions startSparkHistoryServerOptions = new StartSparkHistoryServerOptions.Builder()
+        .instanceId("dc0e9889-eab2-4t9e-9441-566209499546")
+        .build();
+
+      Response<SparkHistoryServerResponse> response = ibmAnalyticsEngineApiService.startSparkHistoryServer(startSparkHistoryServerOptions).execute();
+      SparkHistoryServerResponse sparkHistoryServerResponse = response.getResult();
+
+      System.out.println(sparkHistoryServerResponse);
+      // end-start_spark_history_server
+    } catch (ServiceResponseException e) {
+        logger.error(String.format("Service returned status code %s: %s%nError details: %s",
+          e.getStatusCode(), e.getMessage(), e.getDebuggingInfo()), e);
+    }
+
+    try {
+      System.out.println("getSparkHistoryServer() result:");
+      // begin-get_spark_history_server
+      GetSparkHistoryServerOptions getSparkHistoryServerOptions = new GetSparkHistoryServerOptions.Builder()
+        .instanceId("dc0e9889-eab2-4t9e-9441-566209499546")
+        .build();
+
+      Response<SparkHistoryServerResponse> response = ibmAnalyticsEngineApiService.getSparkHistoryServer(getSparkHistoryServerOptions).execute();
+      SparkHistoryServerResponse sparkHistoryServerResponse = response.getResult();
+
+      System.out.println(sparkHistoryServerResponse);
+      // end-get_spark_history_server
+    } catch (ServiceResponseException e) {
+        logger.error(String.format("Service returned status code %s: %s%nError details: %s",
+          e.getStatusCode(), e.getMessage(), e.getDebuggingInfo()), e);
+    }
+
+    try {
+      // begin-stop_spark_history_server
+      StopSparkHistoryServerOptions stopSparkHistoryServerOptions = new StopSparkHistoryServerOptions.Builder()
+        .instanceId("dc0e9889-eab2-4t9e-9441-566209499546")
+        .build();
+
+      Response<Void> response = ibmAnalyticsEngineApiService.stopSparkHistoryServer(stopSparkHistoryServerOptions).execute();
+      // end-stop_spark_history_server
+      System.out.printf("stopSparkHistoryServer() response status code: %d%n", response.getStatusCode());
+    } catch (ServiceResponseException e) {
+        logger.error(String.format("Service returned status code %s: %s%nError details: %s",
+          e.getStatusCode(), e.getMessage(), e.getDebuggingInfo()), e);
+    }
+
     try {
       // begin-delete_application
       DeleteApplicationOptions deleteApplicationOptions = new DeleteApplicationOptions.Builder()

modules/ibm-analytics-engine-api/src/main/java/com/ibm/cloud/iaesdk/ibm_analytics_engine_api/v3/IbmAnalyticsEngineApi.java

@@ -37,6 +37,7 @@
 import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.GetLogForwardingConfigOptions;
 import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.GetLoggingConfigurationOptions;
 import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.GetResourceConsumptionLimitsOptions;
+import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.GetSparkHistoryServerOptions;
 import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.Instance;
 import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.InstanceGetStateResponse;
 import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.InstanceHomeResponse;
@@ -49,6 +50,9 @@
 import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.ResourceConsumptionLimitsResponse;
 import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.Runtime;
 import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.SetInstanceHomeOptions;
+import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.SparkHistoryServerResponse;
+import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.StartSparkHistoryServerOptions;
+import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.StopSparkHistoryServerOptions;
 import com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model.UpdateInstanceDefaultConfigsOptions;
 import com.ibm.cloud.sdk.core.http.RequestBuilder;
 import com.ibm.cloud.sdk.core.http.ResponseConverter;
@@ -601,6 +605,7 @@ public ServiceCall<LogForwardingConfigResponse> getLogForwardingConfig(GetLogFor
    * Enable or disable log forwarding.
    *
    * Enable or disable log forwarding from IBM Analytics Engine to IBM Log Analysis server.
+   * *Note:* Deprecated. Use the log forwarding config api instead.
    *
    * @param configurePlatformLoggingOptions the {@link ConfigurePlatformLoggingOptions} containing the options for the call
    * @return a {@link ServiceCall} with a result of type {@link LoggingConfigurationResponse}
@@ -630,6 +635,7 @@ public ServiceCall<LoggingConfigurationResponse> configurePlatformLogging(Config
    * Retrieve the logging configuration for a given instance id.
    *
    * Retrieve the logging configuration of a given Analytics Engine instance.
+   * *Note:* Deprecated. Use the log forwarding config api instead.
    *
    * @param getLoggingConfigurationOptions the {@link GetLoggingConfigurationOptions} containing the options for the call
    * @return a {@link ServiceCall} with a result of type {@link LoggingConfigurationResponse}
@@ -650,4 +656,74 @@ public ServiceCall<LoggingConfigurationResponse> getLoggingConfiguration(GetLogg
     return createServiceCall(builder.build(), responseConverter);
   }
 
+  /**
+   * Start Spark history server.
+   *
+   * Start the Spark history server for the given Analytics Engine instance.
+   *
+   * @param startSparkHistoryServerOptions the {@link StartSparkHistoryServerOptions} containing the options for the call
+   * @return a {@link ServiceCall} with a result of type {@link SparkHistoryServerResponse}
+   */
+  public ServiceCall<SparkHistoryServerResponse> startSparkHistoryServer(StartSparkHistoryServerOptions startSparkHistoryServerOptions) {
+    com.ibm.cloud.sdk.core.util.Validator.notNull(startSparkHistoryServerOptions,
+      "startSparkHistoryServerOptions cannot be null");
+    Map<String, String> pathParamsMap = new HashMap<String, String>();
+    pathParamsMap.put("instance_id", startSparkHistoryServerOptions.instanceId());
+    RequestBuilder builder = RequestBuilder.post(RequestBuilder.resolveRequestUrl(getServiceUrl(), "/v3/analytics_engines/{instance_id}/spark_history_server", pathParamsMap));
+    Map<String, String> sdkHeaders = SdkCommon.getSdkHeaders("ibm_analytics_engine_api", "v3", "startSparkHistoryServer");
+    for (Entry<String, String> header : sdkHeaders.entrySet()) {
+      builder.header(header.getKey(), header.getValue());
+    }
+    builder.header("Accept", "application/json");
+    ResponseConverter<SparkHistoryServerResponse> responseConverter =
+      ResponseConverterUtils.getValue(new com.google.gson.reflect.TypeToken<SparkHistoryServerResponse>() { }.getType());
+    return createServiceCall(builder.build(), responseConverter);
+  }
+
+  /**
+   * Get Spark history server details.
+   *
+   * Get the details of the Spark history server of the given Analytics Engine instance.
+   *
+   * @param getSparkHistoryServerOptions the {@link GetSparkHistoryServerOptions} containing the options for the call
+   * @return a {@link ServiceCall} with a result of type {@link SparkHistoryServerResponse}
+   */
+  public ServiceCall<SparkHistoryServerResponse> getSparkHistoryServer(GetSparkHistoryServerOptions getSparkHistoryServerOptions) {
+    com.ibm.cloud.sdk.core.util.Validator.notNull(getSparkHistoryServerOptions,
+      "getSparkHistoryServerOptions cannot be null");
+    Map<String, String> pathParamsMap = new HashMap<String, String>();
+    pathParamsMap.put("instance_id", getSparkHistoryServerOptions.instanceId());
+    RequestBuilder builder = RequestBuilder.get(RequestBuilder.resolveRequestUrl(getServiceUrl(), "/v3/analytics_engines/{instance_id}/spark_history_server", pathParamsMap));
+    Map<String, String> sdkHeaders = SdkCommon.getSdkHeaders("ibm_analytics_engine_api", "v3", "getSparkHistoryServer");
+    for (Entry<String, String> header : sdkHeaders.entrySet()) {
+      builder.header(header.getKey(), header.getValue());
+    }
+    builder.header("Accept", "application/json");
+    ResponseConverter<SparkHistoryServerResponse> responseConverter =
+      ResponseConverterUtils.getValue(new com.google.gson.reflect.TypeToken<SparkHistoryServerResponse>() { }.getType());
+    return createServiceCall(builder.build(), responseConverter);
+  }
+
+  /**
+   * Stop Spark history server.
+   *
+   * Stop the Spark history server of the given Analytics Engine instance.
+   *
+   * @param stopSparkHistoryServerOptions the {@link StopSparkHistoryServerOptions} containing the options for the call
+   * @return a {@link ServiceCall} with a void result
+   */
+  public ServiceCall<Void> stopSparkHistoryServer(StopSparkHistoryServerOptions stopSparkHistoryServerOptions) {
+    com.ibm.cloud.sdk.core.util.Validator.notNull(stopSparkHistoryServerOptions,
+      "stopSparkHistoryServerOptions cannot be null");
+    Map<String, String> pathParamsMap = new HashMap<String, String>();
+    pathParamsMap.put("instance_id", stopSparkHistoryServerOptions.instanceId());
+    RequestBuilder builder = RequestBuilder.delete(RequestBuilder.resolveRequestUrl(getServiceUrl(), "/v3/analytics_engines/{instance_id}/spark_history_server", pathParamsMap));
+    Map<String, String> sdkHeaders = SdkCommon.getSdkHeaders("ibm_analytics_engine_api", "v3", "stopSparkHistoryServer");
+    for (Entry<String, String> header : sdkHeaders.entrySet()) {
+      builder.header(header.getKey(), header.getValue());
+    }
+    ResponseConverter<Void> responseConverter = ResponseConverterUtils.getVoid();
+    return createServiceCall(builder.build(), responseConverter);
+  }
+
 }

modules/ibm-analytics-engine-api/src/main/java/com/ibm/cloud/iaesdk/ibm_analytics_engine_api/v3/model/Application.java

@@ -1,5 +1,5 @@
 /*
- * (C) Copyright IBM Corp. 2022.
+ * (C) Copyright IBM Corp. 2023.
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
  * the License. You may obtain a copy of the License at
@@ -12,6 +12,8 @@
  */
 package com.ibm.cloud.iaesdk.ibm_analytics_engine_api.v3.model;
 
+import java.util.Date;
+
 import com.google.gson.annotations.SerializedName;
 import com.ibm.cloud.sdk.core.service.model.GenericModel;
 
@@ -30,16 +32,8 @@ public interface State {
     String RUNNING = "running";
     /** failed. */
     String FAILED = "failed";
-    /** error. */
-    String ERROR = "error";
     /** accepted. */
     String ACCEPTED = "accepted";
-    /** submitted. */
-    String SUBMITTED = "submitted";
-    /** waiting. */
-    String WAITING = "waiting";
-    /** unknown. */
-    String UNKNOWN = "unknown";
     /** stopped. */
     String STOPPED = "stopped";
     /** auto_terminated. */
@@ -56,12 +50,18 @@ public interface State {
   @SerializedName("spark_application_name")
   protected String sparkApplicationName;
   protected String state;
+  @SerializedName("spark_ui")
+  protected String sparkUi;
+  @SerializedName("submission_time")
+  protected Date submissionTime;
   @SerializedName("start_time")
-  protected String startTime;
+  protected Date startTime;
   @SerializedName("end_time")
-  protected String endTime;
+  protected Date endTime;
   @SerializedName("finish_time")
-  protected String finishTime;
+  protected Date finishTime;
+  @SerializedName("auto_termination_time")
+  protected Date autoTerminationTime;
 
   /**
    * Gets the id.
@@ -129,14 +129,36 @@ public String getState() {
     return state;
   }
 
+  /**
+   * Gets the sparkUi.
+   *
+   * URL of the Apache Spark web UI that is available when the application is running.
+   *
+   * @return the sparkUi
+   */
+  public String getSparkUi() {
+    return sparkUi;
+  }
+
+  /**
+   * Gets the submissionTime.
+   *
+   * Time when the application was submitted.
+   *
+   * @return the submissionTime
+   */
+  public Date getSubmissionTime() {
+    return submissionTime;
+  }
+
   /**
    * Gets the startTime.
    *
    * Time when the application was started.
    *
    * @return the startTime
    */
-  public String getStartTime() {
+  public Date getStartTime() {
     return startTime;
   }
 
@@ -147,19 +169,30 @@ public String getStartTime() {
    *
    * @return the endTime
    */
-  public String getEndTime() {
+  public Date getEndTime() {
     return endTime;
   }
 
   /**
    * Gets the finishTime.
    *
-   * Time when the application was completed.
+   * (deprecated) Time when the application was completed.
    *
    * @return the finishTime
    */
-  public String getFinishTime() {
+  public Date getFinishTime() {
     return finishTime;
   }
+
+  /**
+   * Gets the autoTerminationTime.
+   *
+   * Time when the application will be automatically stopped by the service.
+   *
+   * @return the autoTerminationTime
+   */
+  public Date getAutoTerminationTime() {
+    return autoTerminationTime;
+  }
 }