feat: [dataproc] add support for new Dataproc features

packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/autoscaling_policies.proto

@@ -1,4 +1,4 @@
-// Copyright 2021 Google LLC
+// Copyright 2022 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -36,10 +36,12 @@ option (google.api.resource_definition) = {
 // Dataproc API.
 service AutoscalingPolicyService {
   option (google.api.default_host) = "dataproc.googleapis.com";
-  option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform";
+  option (google.api.oauth_scopes) =
+      "https://www.googleapis.com/auth/cloud-platform";
 
   // Creates new autoscaling policy.
-  rpc CreateAutoscalingPolicy(CreateAutoscalingPolicyRequest) returns (AutoscalingPolicy) {
+  rpc CreateAutoscalingPolicy(CreateAutoscalingPolicyRequest)
+      returns (AutoscalingPolicy) {
     option (google.api.http) = {
       post: "/v1/{parent=projects/*/locations/*}/autoscalingPolicies"
       body: "policy"
@@ -55,7 +57,8 @@ service AutoscalingPolicyService {
   //
   // Disabled check for update_mask, because all updates will be full
   // replacements.
-  rpc UpdateAutoscalingPolicy(UpdateAutoscalingPolicyRequest) returns (AutoscalingPolicy) {
+  rpc UpdateAutoscalingPolicy(UpdateAutoscalingPolicyRequest)
+      returns (AutoscalingPolicy) {
     option (google.api.http) = {
       put: "/v1/{policy.name=projects/*/locations/*/autoscalingPolicies/*}"
       body: "policy"
@@ -68,7 +71,8 @@ service AutoscalingPolicyService {
   }
 
   // Retrieves autoscaling policy.
-  rpc GetAutoscalingPolicy(GetAutoscalingPolicyRequest) returns (AutoscalingPolicy) {
+  rpc GetAutoscalingPolicy(GetAutoscalingPolicyRequest)
+      returns (AutoscalingPolicy) {
     option (google.api.http) = {
       get: "/v1/{name=projects/*/locations/*/autoscalingPolicies/*}"
       additional_bindings {
@@ -79,7 +83,8 @@ service AutoscalingPolicyService {
   }
 
   // Lists autoscaling policies in the project.
-  rpc ListAutoscalingPolicies(ListAutoscalingPoliciesRequest) returns (ListAutoscalingPoliciesResponse) {
+  rpc ListAutoscalingPolicies(ListAutoscalingPoliciesRequest)
+      returns (ListAutoscalingPoliciesResponse) {
     option (google.api.http) = {
       get: "/v1/{parent=projects/*/locations/*}/autoscalingPolicies"
       additional_bindings {
@@ -91,7 +96,8 @@ service AutoscalingPolicyService {
 
   // Deletes an autoscaling policy. It is an error to delete an autoscaling
   // policy that is in use by one or more clusters.
-  rpc DeleteAutoscalingPolicy(DeleteAutoscalingPolicyRequest) returns (google.protobuf.Empty) {
+  rpc DeleteAutoscalingPolicy(DeleteAutoscalingPolicyRequest)
+      returns (google.protobuf.Empty) {
     option (google.api.http) = {
       delete: "/v1/{name=projects/*/locations/*/autoscalingPolicies/*}"
       additional_bindings {
@@ -132,14 +138,17 @@ message AutoscalingPolicy {
 
   // Autoscaling algorithm for policy.
   oneof algorithm {
-    BasicAutoscalingAlgorithm basic_algorithm = 3 [(google.api.field_behavior) = REQUIRED];
+    BasicAutoscalingAlgorithm basic_algorithm = 3
+        [(google.api.field_behavior) = REQUIRED];
   }
 
   // Required. Describes how the autoscaler will operate for primary workers.
-  InstanceGroupAutoscalingPolicyConfig worker_config = 4 [(google.api.field_behavior) = REQUIRED];
+  InstanceGroupAutoscalingPolicyConfig worker_config = 4
+      [(google.api.field_behavior) = REQUIRED];
 
   // Optional. Describes how the autoscaler will operate for secondary workers.
-  InstanceGroupAutoscalingPolicyConfig secondary_worker_config = 5 [(google.api.field_behavior) = OPTIONAL];
+  InstanceGroupAutoscalingPolicyConfig secondary_worker_config = 5
+      [(google.api.field_behavior) = OPTIONAL];
 
   // Optional. The labels to associate with this autoscaling policy.
   // Label **keys** must contain 1 to 63 characters, and must conform to
@@ -155,14 +164,16 @@ message AutoscalingPolicy {
 message BasicAutoscalingAlgorithm {
   oneof config {
     // Required. YARN autoscaling configuration.
-    BasicYarnAutoscalingConfig yarn_config = 1 [(google.api.field_behavior) = REQUIRED];
+    BasicYarnAutoscalingConfig yarn_config = 1
+        [(google.api.field_behavior) = REQUIRED];
   }
 
   // Optional. Duration between scaling events. A scaling period starts after
   // the update operation from the previous event has completed.
   //
   // Bounds: [2m, 1d]. Default: 2m.
-  google.protobuf.Duration cooldown_period = 2 [(google.api.field_behavior) = OPTIONAL];
+  google.protobuf.Duration cooldown_period = 2
+      [(google.api.field_behavior) = OPTIONAL];
 }
 
 // Basic autoscaling configurations for YARN.
@@ -173,22 +184,23 @@ message BasicYarnAutoscalingConfig {
   // downscaling operations.
   //
   // Bounds: [0s, 1d].
-  google.protobuf.Duration graceful_decommission_timeout = 5 [(google.api.field_behavior) = REQUIRED];
-
-  // Required. Fraction of average YARN pending memory in the last cooldown period
-  // for which to add workers. A scale-up factor of 1.0 will result in scaling
-  // up so that there is no pending memory remaining after the update (more
-  // aggressive scaling). A scale-up factor closer to 0 will result in a smaller
-  // magnitude of scaling up (less aggressive scaling).
-  // See [How autoscaling
+  google.protobuf.Duration graceful_decommission_timeout = 5
+      [(google.api.field_behavior) = REQUIRED];
+
+  // Required. Fraction of average YARN pending memory in the last cooldown
+  // period for which to add workers. A scale-up factor of 1.0 will result in
+  // scaling up so that there is no pending memory remaining after the update
+  // (more aggressive scaling). A scale-up factor closer to 0 will result in a
+  // smaller magnitude of scaling up (less aggressive scaling). See [How
+  // autoscaling
   // works](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/autoscaling#how_autoscaling_works)
   // for more information.
   //
   // Bounds: [0.0, 1.0].
   double scale_up_factor = 1 [(google.api.field_behavior) = REQUIRED];
 
-  // Required. Fraction of average YARN pending memory in the last cooldown period
-  // for which to remove workers. A scale-down factor of 1 will result in
+  // Required. Fraction of average YARN pending memory in the last cooldown
+  // period for which to remove workers. A scale-down factor of 1 will result in
   // scaling down so that there is no available memory remaining after the
   // update (more aggressive scaling). A scale-down factor of 0 disables
   // removing workers, which can be beneficial for autoscaling a single job.
@@ -206,7 +218,8 @@ message BasicYarnAutoscalingConfig {
   // on any recommended change.
   //
   // Bounds: [0.0, 1.0]. Default: 0.0.
-  double scale_up_min_worker_fraction = 3 [(google.api.field_behavior) = OPTIONAL];
+  double scale_up_min_worker_fraction = 3
+      [(google.api.field_behavior) = OPTIONAL];
 
   // Optional. Minimum scale-down threshold as a fraction of total cluster size
   // before scaling occurs. For example, in a 20-worker cluster, a threshold of
@@ -215,7 +228,8 @@ message BasicYarnAutoscalingConfig {
   // on any recommended change.
   //
   // Bounds: [0.0, 1.0]. Default: 0.0.
-  double scale_down_min_worker_fraction = 4 [(google.api.field_behavior) = OPTIONAL];
+  double scale_down_min_worker_fraction = 4
+      [(google.api.field_behavior) = OPTIONAL];
 }
 
 // Configuration for the size bounds of an instance group, including its
@@ -358,7 +372,8 @@ message ListAutoscalingPoliciesRequest {
 // A response to a request to list autoscaling policies in a project.
 message ListAutoscalingPoliciesResponse {
   // Output only. Autoscaling policies list.
-  repeated AutoscalingPolicy policies = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+  repeated AutoscalingPolicy policies = 1
+      [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Output only. This token is included in the response if there are more
   // results to fetch.

packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/batches.proto

@@ -1,4 +1,4 @@
-// Copyright 2021 Google LLC
+// Copyright 2022 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -33,7 +33,8 @@ option java_package = "com.google.cloud.dataproc.v1";
 // The BatchController provides methods to manage batch workloads.
 service BatchController {
   option (google.api.default_host) = "dataproc.googleapis.com";
-  option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform";
+  option (google.api.oauth_scopes) =
+      "https://www.googleapis.com/auth/cloud-platform";
 
   // Creates a batch workload that executes asynchronously.
   rpc CreateBatch(CreateBatchRequest) returns (google.longrunning.Operation) {
@@ -87,8 +88,8 @@ message CreateBatchRequest {
   // Required. The batch to create.
   Batch batch = 2 [(google.api.field_behavior) = REQUIRED];
 
-  // Optional. The ID to use for the batch, which will become the final component of
-  // the batch's resource name.
+  // Optional. The ID to use for the batch, which will become the final
+  // component of the batch's resource name.
   //
   // This value must be 4-63 characters. Valid characters are `/[a-z][0-9]-/`.
   string batch_id = 3 [(google.api.field_behavior) = OPTIONAL];
@@ -110,12 +111,12 @@ message CreateBatchRequest {
 
 // A request to get the resource representation for a batch workload.
 message GetBatchRequest {
-  // Required. The name of the batch to retrieve.
+  // Required. The fully qualified name of the batch to retrieve
+  // in the format
+  // "projects/PROJECT_ID/locations/DATAPROC_REGION/batches/BATCH_ID"
   string name = 1 [
     (google.api.field_behavior) = REQUIRED,
-    (google.api.resource_reference) = {
-      type: "dataproc.googleapis.com/Batch"
-    }
+    (google.api.resource_reference) = { type: "dataproc.googleapis.com/Batch" }
   ];
 }
 
@@ -137,6 +138,28 @@ message ListBatchesRequest {
   // Optional. A page token received from a previous `ListBatches` call.
   // Provide this token to retrieve the subsequent page.
   string page_token = 3 [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. A filter for the batches to return in the response.
+  //
+  // A filter is a logical expression constraining the values of various fields
+  // in each batch resource. Filters are case sensitive, and may contain
+  // multiple clauses combined with logical operators (AND/OR).
+  // Supported fields are `batch_id`, `batch_uuid`, `state`, and `create_time`.
+  //
+  // e.g. `state = RUNNING and create_time < "2023-01-01T00:00:00Z"`
+  // filters for batches in state RUNNING that were created before 2023-01-01
+  //
+  // See https://google.aip.dev/assets/misc/ebnf-filtering.txt for a detailed
+  // description of the filter syntax and a list of supported comparisons.
+  string filter = 4 [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. Field(s) on which to sort the list of batches.
+  //
+  // Currently the only supported sort orders are unspecified (empty) and
+  // `create_time desc` to sort by most recently created batches first.
+  //
+  // See https://google.aip.dev/132#ordering for more details.
+  string order_by = 5 [(google.api.field_behavior) = OPTIONAL];
 }
 
 // A list of batch workloads.
@@ -151,12 +174,12 @@ message ListBatchesResponse {
 
 // A request to delete a batch workload.
 message DeleteBatchRequest {
-  // Required. The name of the batch resource to delete.
+  // Required. The fully qualified name of the batch to retrieve
+  // in the format
+  // "projects/PROJECT_ID/locations/DATAPROC_REGION/batches/BATCH_ID"
   string name = 1 [
     (google.api.field_behavior) = REQUIRED,
-    (google.api.resource_reference) = {
-      type: "dataproc.googleapis.com/Batch"
-    }
+    (google.api.resource_reference) = { type: "dataproc.googleapis.com/Batch" }
   ];
 }
 
@@ -167,18 +190,6 @@ message Batch {
     pattern: "projects/{project}/locations/{location}/batches/{batch}"
   };
 
-  // Historical state information.
-  message StateHistory {
-    // Output only. The state of the batch at this point in history.
-    State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
-
-    // Output only. Details about the state at this point in history.
-    string state_message = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
-
-    // Output only. The time when the batch entered the historical state.
-    google.protobuf.Timestamp state_start_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
-  }
-
   // The batch state.
   enum State {
     // The batch state is unknown.
@@ -203,6 +214,19 @@ message Batch {
     FAILED = 6;
   }
 
+  // Historical state information.
+  message StateHistory {
+    // Output only. The state of the batch at this point in history.
+    State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+    // Output only. Details about the state at this point in history.
+    string state_message = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+    // Output only. The time when the batch entered the historical state.
+    google.protobuf.Timestamp state_start_time = 3
+        [(google.api.field_behavior) = OUTPUT_ONLY];
+  }
+
   // Output only. The resource name of the batch.
   string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
 
@@ -211,7 +235,8 @@ message Batch {
   string uuid = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Output only. The time when the batch was created.
-  google.protobuf.Timestamp create_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
+  google.protobuf.Timestamp create_time = 3
+      [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // The application/framework-specific portion of the batch configuration.
   oneof batch_config {
@@ -239,7 +264,8 @@ message Batch {
   string state_message = 10 [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Output only. The time when the batch entered a current state.
-  google.protobuf.Timestamp state_time = 11 [(google.api.field_behavior) = OUTPUT_ONLY];
+  google.protobuf.Timestamp state_time = 11
+      [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Output only. The email address of the user who created the batch.
   string creator = 12 [(google.api.field_behavior) = OUTPUT_ONLY];
@@ -257,22 +283,24 @@ message Batch {
   RuntimeConfig runtime_config = 14 [(google.api.field_behavior) = OPTIONAL];
 
   // Optional. Environment configuration for the batch execution.
-  EnvironmentConfig environment_config = 15 [(google.api.field_behavior) = OPTIONAL];
+  EnvironmentConfig environment_config = 15
+      [(google.api.field_behavior) = OPTIONAL];
 
   // Output only. The resource name of the operation associated with this batch.
   string operation = 16 [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Output only. Historical state information for the batch.
-  repeated StateHistory state_history = 17 [(google.api.field_behavior) = OUTPUT_ONLY];
+  repeated StateHistory state_history = 17
+      [(google.api.field_behavior) = OUTPUT_ONLY];
 }
 
 // A configuration for running an
 // [Apache
 // PySpark](https://spark.apache.org/docs/latest/api/python/getting_started/quickstart.html)
 // batch workload.
 message PySparkBatch {
-  // Required. The HCFS URI of the main Python file to use as the Spark driver. Must
-  // be a .py file.
+  // Required. The HCFS URI of the main Python file to use as the Spark driver.
+  // Must be a .py file.
   string main_python_file_uri = 1 [(google.api.field_behavior) = REQUIRED];
 
   // Optional. The arguments to pass to the driver. Do not include arguments
@@ -298,7 +326,7 @@ message PySparkBatch {
   repeated string archive_uris = 6 [(google.api.field_behavior) = OPTIONAL];
 }
 
-// A configuration for running an [Apache Spark](http://spark.apache.org/)
+// A configuration for running an [Apache Spark](https://spark.apache.org/)
 // batch workload.
 message SparkBatch {
   // The specification of the main method to call to drive the Spark
@@ -310,8 +338,8 @@ message SparkBatch {
     // Optional. The HCFS URI of the jar file that contains the main class.
     string main_jar_file_uri = 1 [(google.api.field_behavior) = OPTIONAL];
 
-    // Optional. The name of the driver main class. The jar file that contains the class
-    // must be in the classpath or specified in `jar_file_uris`.
+    // Optional. The name of the driver main class. The jar file that contains
+    // the class must be in the classpath or specified in `jar_file_uris`.
     string main_class = 2 [(google.api.field_behavior) = OPTIONAL];
   }
 
@@ -342,9 +370,9 @@ message SparkRBatch {
   // Must be a `.R` or `.r` file.
   string main_r_file_uri = 1 [(google.api.field_behavior) = REQUIRED];
 
-  // Optional. The arguments to pass to the Spark driver. Do not include arguments
-  // that can be set as batch properties, such as `--conf`, since a collision
-  // can occur that causes an incorrect batch submission.
+  // Optional. The arguments to pass to the Spark driver. Do not include
+  // arguments that can be set as batch properties, such as `--conf`, since a
+  // collision can occur that causes an incorrect batch submission.
   repeated string args = 2 [(google.api.field_behavior) = OPTIONAL];
 
   // Optional. HCFS URIs of files to be placed in the working directory of
@@ -358,14 +386,17 @@ message SparkRBatch {
 }
 
 // A configuration for running
-// [Apache Spark SQL](http://spark.apache.org/sql/) queries as a batch workload.
+// [Apache Spark SQL](https://spark.apache.org/sql/) queries as a batch
+// workload.
 message SparkSqlBatch {
-  // Required. The HCFS URI of the script that contains Spark SQL queries to execute.
+  // Required. The HCFS URI of the script that contains Spark SQL queries to
+  // execute.
   string query_file_uri = 1 [(google.api.field_behavior) = REQUIRED];
 
   // Optional. Mapping of query variable names to values (equivalent to the
   // Spark SQL command: `SET name="value";`).
-  map<string, string> query_variables = 2 [(google.api.field_behavior) = OPTIONAL];
+  map<string, string> query_variables = 2
+      [(google.api.field_behavior) = OPTIONAL];
 
   // Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.
   repeated string jar_file_uris = 3 [(google.api.field_behavior) = OPTIONAL];