add export to bq feature (#2281)

* Subject: add export to BigQuery  Type: new feature

* Fix comments on passing args

* fix copyright time

* fix gethistory failure

Co-authored-by: Les Vogel <[email protected]>

Author: 2 people

asset/src/main/java/com/example/asset/ExportAssetsBigqueryExample.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.example.asset;
+
+// [START asset_quickstart_export_assets_bigquery]
+// Imports the Google Cloud client library
+
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.asset.v1.AssetServiceClient;
+import com.google.cloud.asset.v1.BigQueryDestination;
+import com.google.cloud.asset.v1.ExportAssetsRequest;
+import com.google.cloud.asset.v1.ExportAssetsResponse;
+import com.google.cloud.asset.v1.OutputConfig;
+import com.google.cloud.asset.v1.ProjectName;
+
+public class ExportAssetsBigqueryExample {
+
+  // Use the default project Id.
+  private static final String projectId = ServiceOptions.getDefaultProjectId();
+
+  // Export assets for a project.
+  // @param args path where the results will be exported to.
+  public static void exportBigQuery(String bigqueryDataset, String bigqueryTable) throws Exception {
+    try (AssetServiceClient client = AssetServiceClient.create()) {
+      ProjectName parent = ProjectName.of(projectId);
+      OutputConfig outputConfig =
+          OutputConfig.newBuilder()
+              .setBigqueryDestination(
+                  BigQueryDestination.newBuilder()
+                      .setDataset(bigqueryDataset)
+                      .setTable(bigqueryTable)
+                      .setForce(true)
+                      .build())
+              .build();
+      ExportAssetsRequest request =
+          ExportAssetsRequest.newBuilder()
+              .setParent(parent.toString())
+              .setOutputConfig(outputConfig)
+              .build();
+      ExportAssetsResponse response = client.exportAssetsAsync(request).get();
+      System.out.println(response);
+    }
+  }
+}
+// [END asset_quickstart_export_assets_bigquery]

asset/src/test/java/com/example/asset/QuickStartIT.java

@@ -18,6 +18,14 @@
 
 import static com.google.common.truth.Truth.assertThat;
 
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQuery;
+import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.Dataset;
+import com.google.cloud.bigquery.DatasetId;
+import com.google.cloud.bigquery.DatasetInfo;
+import com.google.cloud.bigquery.testing.RemoteBigQueryHelper;
 import com.google.cloud.storage.BlobInfo;
 import com.google.cloud.storage.Bucket;
 import com.google.cloud.storage.BucketInfo;
@@ -38,8 +46,10 @@
 @SuppressWarnings("checkstyle:abbreviationaswordinname")
 public class QuickStartIT {
   private static final String bucketName = UUID.randomUUID().toString();
+  private static final String datasetName = RemoteBigQueryHelper.generateDatasetName();
   private ByteArrayOutputStream bout;
   private PrintStream out;
+  private BigQuery bigquery;
 
   private static final void deleteBucket(String bucketName) {
     Storage storage = StorageOptions.getDefaultInstance().getService();
@@ -62,6 +72,10 @@ private static final void createBucket(String bucketName) {
   @Before
   public void setUp() {
     createBucket(bucketName);
+    bigquery = BigQueryOptions.getDefaultInstance().getService();
+    if (bigquery.getDataset(datasetName) == null) {
+      Dataset dataset = bigquery.create(DatasetInfo.newBuilder(datasetName).build());
+    }
     bout = new ByteArrayOutputStream();
     out = new PrintStream(bout);
     System.setOut(out);
@@ -72,6 +86,8 @@ public void tearDown() {
     String consoleOutput = bout.toString();
     System.setOut(null);
     deleteBucket(bucketName);
+    DatasetId datasetId = DatasetId.of(bigquery.getOptions().getProjectId(), datasetName);
+    bigquery.delete(datasetId, DatasetDeleteOption.deleteContents());
   }
 
   @Test
@@ -82,8 +98,20 @@ public void testExportAssetExample() throws Exception {
     assertThat(got).contains(String.format("uri: \"%s\"", assetDumpPath));
   }
 
+  @Test
+  public void testExportAssetBigqueryExample() throws Exception {
+    String dataset =
+        String.format("projects/%s/datasets/%s", ServiceOptions.getDefaultProjectId(), datasetName);
+    String table = "java_test";
+    ExportAssetsBigqueryExample.exportBigQuery(dataset, table);
+    String got = bout.toString();
+    assertThat(got).contains(String.format("dataset: \"%s\"", dataset));
+  }
+
   @Test
   public void testBatchGetAssetsHistory() throws Exception {
+    // Wait 10 seconds to let bucket creation event go to CAI
+    Thread.sleep(10000);
     String bucketAssetName = String.format("//storage.googleapis.com/%s", bucketName);
     BatchGetAssetsHistoryExample.main(bucketAssetName);
     String got = bout.toString();