feat: enable experimental grpc support

.github/.OwlBot.yaml

@@ -25,8 +25,12 @@ deep-copy-regex:
   dest: "/owl-bot-staging/$1/proto-google-cloud-datastore-$1/src"
 - source: "/google/datastore/admin/(v.*)/.*-java/proto-google-.*/src"
   dest: "/owl-bot-staging/$1/proto-google-cloud-datastore-admin-$1/src"
+- source: "/google/datastore/(v.*)/.*-java/grpc-google-.*/src"
+  dest: "/owl-bot-staging/$1/grpc-google-cloud-datastore-$1/src"
 - source: "/google/datastore/admin/(v.*)/.*-java/grpc-google-.*/src"
   dest: "/owl-bot-staging/$1/grpc-google-cloud-datastore-admin-$1/src"
 # Admin & Data APIs share the same wrapper library.
+- source: "/google/datastore/(v.*)/.*-java/gapic-google-.*/src"
+  dest: "/owl-bot-staging/$1/google-cloud-datastore/src"
 - source: "/google/datastore/admin/(v.*)/.*-java/gapic-google-.*/src"
-  dest: "/owl-bot-staging/$1/google-cloud-datastore/src"
+  dest: "/owl-bot-staging/$1/google-cloud-datastore/src"

.github/dependabot.yml

@@ -1,5 +1,4 @@
 version: 2
-
 updates:
   - package-ecosystem: "maven"
     directory: "/"
@@ -18,14 +17,3 @@ updates:
     # it via template in the synthtool repository.
     ignore:
       - dependency-name: "*"
-
-  # rules for the `V3-experimental` branch
-  - package-ecosystem: maven
-    directory: "/"
-    schedule:
-      interval: daily
-    commit-message:
-      # Prefix all commit messages with "deps: "
-      prefix: "deps"
-    open-pull-requests-limit: 10
-    target-branch: "V3-experimental"

.gitignore

@@ -3,4 +3,4 @@ target/
 *.iml
 __pycache__/
 
-.flattened-pom.xml
+.flattened-pom.xml

google-cloud-datastore/clirr-ignored-differences.xml

@@ -1,7 +1,50 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!-- see http://mojo.codehaus.org/clirr-maven-plugin/examples/ignored-differences.html -->
 <differences>
-    <!--  added a default method to an interface-->
+    <!-- renaming internal only class-->
+    <difference>
+        <className>com/google/cloud/datastore/ReadOption$QueryAndReadOptions</className>
+        <method>*</method>
+        <differenceType>8001</differenceType>
+    </difference>
+    <difference>
+        <className>com/google/cloud/datastore/execution/request/AggregationQueryRequestProtoPreparer</className>
+        <method>*QueryAndReadOptions*</method>
+        <to>*QueryConfig*</to>
+        <differenceType>7005</differenceType>
+    </difference>
+    <difference>
+        <!-- Added BaseGrpcServiceException as a superclass -->
+        <className>com/google/cloud/datastore/DatastoreException</className>
+        <differenceType>5000</differenceType>
+        <to>com/google/cloud/grpc/BaseGrpcServiceException</to>
+    </difference>
+    <difference>
+        <!-- Removed BaseHttpServiceException as a superclass -->
+        <className>com/google/cloud/datastore/DatastoreException</className>
+        <differenceType>5001</differenceType>
+        <to>com/google/cloud/http/BaseHttpServiceException</to>
+    </difference>
+    <difference>
+        <className>com/google/cloud/datastore/Datastore</className>
+        <method>void close()</method>
+        <differenceType>7012</differenceType>
+    </difference>
+    <difference>
+        <className>com/google/cloud/datastore/spi/v1/DatastoreRpc</className>
+        <method>void close()</method>
+        <differenceType>7012</differenceType>
+    </difference>
+    <difference>
+        <className>com/google/cloud/datastore/Datastore</className>
+        <method>boolean isClosed()</method>
+        <differenceType>7012</differenceType>
+    </difference>
+    <difference>
+        <className>com/google/cloud/datastore/spi/v1/DatastoreRpc</className>
+        <method>boolean isClosed()</method>
+        <differenceType>7012</differenceType>
+    </difference>
     <difference>
         <className>com/google/cloud/datastore/Datastore</className>
         <method>com.google.cloud.datastore.QueryResults run(com.google.cloud.datastore.Query, com.google.cloud.datastore.models.ExplainOptions, com.google.cloud.datastore.ReadOption[])</method>
@@ -14,7 +57,7 @@
     </difference>
     <difference>
         <className>com/google/cloud/datastore/DatastoreReader</className>
-        <method> com.google.cloud.datastore.AggregationResults runAggregation(com.google.cloud.datastore.AggregationQuery, com.google.cloud.datastore.models.ExplainOptions)</method>
+        <method>com.google.cloud.datastore.AggregationResults runAggregation(com.google.cloud.datastore.AggregationQuery, com.google.cloud.datastore.models.ExplainOptions)</method>
         <differenceType>7012</differenceType>
     </difference>
     <difference>
@@ -28,18 +71,6 @@
         <differenceType>7012</differenceType>
     </difference>
 
-    <!-- changed parmeters on internal only class-->
-    <difference>
-        <className>com/google/cloud/datastore/ReadOption$QueryConfig</className>
-        <method>com.google.cloud.datastore.ReadOption$QueryConfig create(com.google.cloud.datastore.Query, java.util.List)</method>
-        <to>*com.google.datastore.v1.ExplainOptions*</to>
-        <differenceType>7005</differenceType>
-    </difference>
-    <difference>
-        <className>com/google/cloud/datastore/ReadOption$QueryConfig</className>
-        <method>com.google.cloud.datastore.ReadOption$QueryConfig create(com.google.cloud.datastore.Query)</method>
-        <differenceType>7004</differenceType>
-    </difference>
     <difference>
         <className>com/google/cloud/datastore/execution/AggregationQueryExecutor</className>
         <method>com.google.cloud.datastore.AggregationResults execute(com.google.cloud.datastore.AggregationQuery, com.google.cloud.datastore.ReadOption[])</method>

google-cloud-datastore/pom.xml

@@ -18,6 +18,10 @@
     <site.installationModule>google-cloud-datastore</site.installationModule>
   </properties>
   <dependencies>
+    <dependency>
+      <groupId>com.google.api.grpc</groupId>
+      <artifactId>grpc-google-cloud-datastore-v1</artifactId>
+    </dependency>
     <dependency>
       <groupId>com.google.api.grpc</groupId>
       <artifactId>grpc-google-cloud-datastore-admin-v1</artifactId>
@@ -26,6 +30,10 @@
       <groupId>com.google.cloud</groupId>
       <artifactId>google-cloud-core-http</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-core-grpc</artifactId>
+    </dependency>
     <dependency>
       <groupId>com.google.api.grpc</groupId>
       <artifactId>proto-google-cloud-datastore-v1</artifactId>

google-cloud-datastore/src/main/java/com/google/cloud/datastore/Batch.java

@@ -16,6 +16,7 @@
 
 package com.google.cloud.datastore;
 
+import com.google.api.core.InternalExtensionOnly;
 import java.util.List;
 import javax.annotation.concurrent.NotThreadSafe;
 
@@ -42,6 +43,7 @@
  * This class too should not be treated as a thread safe class. </b>
  */
 @NotThreadSafe
+@InternalExtensionOnly
 public interface Batch extends DatastoreBatchWriter {
 
   interface Response {

google-cloud-datastore/src/main/java/com/google/cloud/datastore/Datastore.java

@@ -17,14 +17,16 @@
 package com.google.cloud.datastore;
 
 import com.google.api.core.BetaApi;
+import com.google.api.core.InternalExtensionOnly;
 import com.google.cloud.Service;
 import com.google.cloud.datastore.models.ExplainOptions;
 import com.google.datastore.v1.TransactionOptions;
 import java.util.Iterator;
 import java.util.List;
 
 /** An interface for Google Cloud Datastore. */
-public interface Datastore extends Service<DatastoreOptions>, DatastoreReaderWriter {
+@InternalExtensionOnly
+public interface Datastore extends Service<DatastoreOptions>, DatastoreReaderWriter, AutoCloseable {
 
   /**
    * Returns a new Datastore transaction.
@@ -51,9 +53,9 @@ public interface Datastore extends Service<DatastoreOptions>, DatastoreReaderWri
    * @param <T> the type of the return value
    */
   interface TransactionCallable<T> {
+
     T run(DatastoreReaderWriter readerWriter) throws Exception;
   }
-
   /**
    * Invokes the callback's {@link Datastore.TransactionCallable#run} method with a {@link
    * DatastoreReaderWriter} that is associated with a new transaction. The transaction will be
@@ -481,10 +483,7 @@ interface TransactionCallable<T> {
    * @throws DatastoreException upon failure
    */
   @BetaApi
-  default <T> QueryResults<T> run(
-      Query<T> query, ExplainOptions explainOptions, ReadOption... options) {
-    throw new UnsupportedOperationException("Not implemented.");
-  }
+  <T> QueryResults<T> run(Query<T> query, ExplainOptions explainOptions, ReadOption... options);
 
   /**
    * Submits a {@link AggregationQuery} and returns {@link AggregationResults}. {@link ReadOption}s
@@ -529,9 +528,7 @@ default <T> QueryResults<T> run(
    * @throws DatastoreException upon failure
    * @return {@link AggregationResults}
    */
-  default AggregationResults runAggregation(AggregationQuery query, ReadOption... options) {
-    throw new UnsupportedOperationException("Not implemented.");
-  }
+  AggregationResults runAggregation(AggregationQuery query, ReadOption... options);
 
   /**
    * Submits a {@link AggregationQuery} with specified {@link
@@ -557,8 +554,17 @@ default AggregationResults runAggregation(AggregationQuery query, ReadOption...
    * @return {@link AggregationResults}
    */
   @BetaApi
-  default AggregationResults runAggregation(
-      AggregationQuery query, ExplainOptions explainOptions, ReadOption... options) {
-    throw new UnsupportedOperationException("Not implemented.");
-  }
+  AggregationResults runAggregation(
+      AggregationQuery query, ExplainOptions explainOptions, ReadOption... options);
+
+  /**
+   * Closes the gRPC channels associated with this instance and frees up their resources. This
+   * method blocks until all channels are closed. Once this method is called, this Datastore client
+   * is no longer usable.
+   */
+  @Override
+  void close() throws Exception;
+
+  /** Returns true if this background resource has been shut down. */
+  boolean isClosed();
 }

google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreBatchWriter.java

@@ -16,6 +16,7 @@
 
 package com.google.cloud.datastore;
 
+import com.google.api.core.InternalExtensionOnly;
 import java.util.List;
 import javax.annotation.concurrent.NotThreadSafe;
 
@@ -31,6 +32,7 @@
  * This class too should not be treated as a thread safe class. </b>
  */
 @NotThreadSafe
+@InternalExtensionOnly
 public interface DatastoreBatchWriter extends DatastoreWriter {
 
   /**

google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreException.java

@@ -16,10 +16,16 @@
 
 package com.google.cloud.datastore;
 
+import com.google.api.gax.grpc.GrpcStatusCode;
+import com.google.api.gax.rpc.ApiException;
+import com.google.api.gax.rpc.StatusCode;
 import com.google.cloud.BaseServiceException;
 import com.google.cloud.RetryHelper.RetryHelperException;
 import com.google.cloud.http.BaseHttpServiceException;
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableSet;
+import io.grpc.StatusException;
+import io.grpc.StatusRuntimeException;
 import java.io.IOException;
 import java.util.Set;
 
@@ -43,6 +49,10 @@ public DatastoreException(int code, String message, String reason) {
     this(code, message, reason, true, null);
   }
 
+  public DatastoreException(int code, String message, Throwable cause) {
+    super(code, message, null, true, RETRYABLE_ERRORS, cause);
+  }
+
   public DatastoreException(int code, String message, String reason, Throwable cause) {
     super(code, message, reason, true, RETRYABLE_ERRORS, cause);
   }
@@ -64,7 +74,76 @@ public DatastoreException(IOException exception) {
    */
   static DatastoreException translateAndThrow(RetryHelperException ex) {
     BaseServiceException.translate(ex);
-    throw new DatastoreException(UNKNOWN_CODE, ex.getMessage(), null, ex.getCause());
+    throw transformThrowable(ex);
+  }
+
+  static BaseServiceException transformThrowable(Throwable t) {
+    if (t instanceof BaseServiceException) {
+      return (BaseServiceException) t;
+    }
+    if (t.getCause() instanceof BaseServiceException) {
+      return (BaseServiceException) t.getCause();
+    }
+    if (t instanceof ApiException) {
+      return asDatastoreException((ApiException) t);
+    }
+    if (t.getCause() instanceof ApiException) {
+      return asDatastoreException((ApiException) t.getCause());
+    }
+    return getDatastoreException(t);
+  }
+
+  private static DatastoreException getDatastoreException(Throwable t) {
+    // unwrap a RetryHelperException if that is what is being translated
+    if (t instanceof RetryHelperException) {
+      return new DatastoreException(UNKNOWN_CODE, t.getMessage(), null, t.getCause());
+    }
+    return new DatastoreException(UNKNOWN_CODE, t.getMessage(), t);
+  }
+
+  static DatastoreException asDatastoreException(ApiException apiEx) {
+    int datastoreStatusCode = 0;
+    StatusCode statusCode = apiEx.getStatusCode();
+    if (statusCode instanceof GrpcStatusCode) {
+      GrpcStatusCode gsc = (GrpcStatusCode) statusCode;
+      datastoreStatusCode =
+          GrpcToDatastoreCodeTranslation.grpcCodeToDatastoreStatusCode(gsc.getTransportCode());
+    }
+
+    // If there is a gRPC exception in our cause, pull its error message up to be our
+    // message otherwise, create a generic error message with the status code.
+    String statusCodeName = statusCode.getCode().name();
+    String statusExceptionMessage = getStatusExceptionMessage(apiEx);
+
+    String message;
+    if (statusExceptionMessage != null) {
+      message = statusCodeName + ": " + statusExceptionMessage;
+    } else {
+      message = "Error: " + statusCodeName;
+    }
+
+    String reason = "";
+    if (Strings.isNullOrEmpty(apiEx.getReason())) {
+      if (apiEx.getStatusCode() != null) {
+        reason = apiEx.getStatusCode().getCode().name();
+      }
+    }
+    // It'd be better to use ExceptionData and BaseServiceException#<init>(ExceptionData) but,
+    // BaseHttpServiceException does not pass that through so we're stuck using this for now.
+    // TODO: When we can break the coupling to BaseHttpServiceException replace this
+    return new DatastoreException(datastoreStatusCode, message, reason, apiEx);
+  }
+
+  private static String getStatusExceptionMessage(Exception apiEx) {
+    if (apiEx.getMessage() != null) {
+      return apiEx.getMessage();
+    } else {
+      Throwable cause = apiEx.getCause();
+      if (cause instanceof StatusRuntimeException || cause instanceof StatusException) {
+        return cause.getMessage();
+      }
+      return null;
+    }
   }
 
   /**

google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreFactory.java

@@ -16,7 +16,9 @@
 
 package com.google.cloud.datastore;
 
+import com.google.api.core.InternalExtensionOnly;
 import com.google.cloud.ServiceFactory;
 
 /** An interface for Datastore factories. */
+@InternalExtensionOnly
 public interface DatastoreFactory extends ServiceFactory<Datastore, DatastoreOptions> {}