From cc8de4abcb604f91cd8b2513fbc8bb3592e2b464 Mon Sep 17 00:00:00 2001 From: Daniel Tang Date: Fri, 16 Sep 2016 11:09:19 -0700 Subject: [PATCH 01/28] add bigquery snippets and tests (in progress) --- .../bigquery/snippets/TableSnippets.java | 115 ++++++++++++++++++ .../bigquery/snippets/ITTableSnippets.java | 68 +++++++++++ 2 files changed, 183 insertions(+) create mode 100644 google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java create mode 100644 google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java new file mode 100644 index 000000000000..c7611246766d --- /dev/null +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java @@ -0,0 +1,115 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.examples.bigquery.snippets; + +import com.google.cloud.Page; +import com.google.cloud.bigquery.BigQuery.TableDataListOption; +import com.google.cloud.bigquery.FieldValue; +import com.google.cloud.bigquery.InsertAllRequest.RowToInsert; +import com.google.cloud.bigquery.InsertAllResponse; +import com.google.cloud.bigquery.Job; +import com.google.cloud.bigquery.Table; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/* + * EDITING INSTRUCTIONS + * This file is referenced in Table’s javadoc. Any change to this file should be reflected in + * Table’s javadoc. + */ +public class TableSnippets { + private final Table table; + + public TableSnippets(Table table) { + this.table = table; + } + + /** + * Example of inserting rows into a table. + */ + // [TARGET insert(Iterable)] + // [VARIABLE "rowId1"] + // [VARIABLE "rowId2"] + public InsertAllResponse insert(String rowId1, String rowId2) { + // [START insert] + List rows = new ArrayList<>(); + Map row1 = new HashMap<>(); + row1.put("stringField", "value1"); + row1.put("booleanField", true); + Map row2 = new HashMap<>(); + row2.put("stringField", "value2"); + row2.put("booleanField", false); + rows.add(RowToInsert.of(rowId1, row1)); + rows.add(RowToInsert.of(rowId2, row2)); + InsertAllResponse response = table.insert(rows); + // do something with response + // [END insert] + return response; + } + + /** + * Example of inserting rows into a table which ignores invalid rows. + */ + // [TARGET insert(Iterable, boolean, boolean)] + // [VARIABLE "rowId1"] + // [VARIABLE "rowId2"] + public InsertAllResponse insertWithParams(String rowId1, String rowId2) { + // [START insertWithParams] + List rows = new ArrayList<>(); + Map row1 = new HashMap<>(); + row1.put("stringField", "value1"); + row1.put("booleanField", true); + Map row2 = new HashMap<>(); + row2.put("stringField", "value2"); + row2.put("booleanField", false); + rows.add(RowToInsert.of(rowId1, row1)); + rows.add(RowToInsert.of(rowId2, row2)); + InsertAllResponse response = table.insert(rows, true, true); + // do something with response + // [END insertWithParams] + return response; + } + + /** + * Example of getting a paginated list of rows in a table. + */ + // [TARGET list(BigQuery.TableDataListOption...)] + public Page> list() { + // [START list] + Page> page = table.list(TableDataListOption.pageSize(100)); + // do something with page + // [END list] + return page; + } + + /** + * Example of copying a table to a destination table and dataset referenced by name. + */ + // [TARGET copy(String, String, BigQuery.JobOption...)] + // [VARIABLE "my_dataset"] + // [VARIABLE "my_destination_table"] + public Job copy(String datasetName, String tableName) { + // [START copy] + Job job = table.copy(datasetName, tableName); + // do something with job + // [END copy] + return job; + } +} \ No newline at end of file diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java new file mode 100644 index 000000000000..eb341ee56072 --- /dev/null +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java @@ -0,0 +1,68 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.examples.bigquery.snippets; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.Field.Type; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardTableDefinition; +import com.google.cloud.bigquery.Table; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableInfo; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.logging.Logger; + +/** + * Integration tests for {@link TableSnippets}. + */ +public class ITTableSnippets { + private static final String TABLE_NAME = "table"; + private static final String DATASET_NAME = "dataset"; + public static final TableId TABLE_ID = TableId.of(DATASET_NAME, TABLE_NAME); + private static final Logger log = Logger.getLogger(ITTableSnippets.class.getName()); + + private static BigQuery bq; + private static Table table; + private static TableSnippets tableSnippets; + + @BeforeClass + public static void beforeClass() { + bq = BigQueryOptions.defaultInstance().service(); + StandardTableDefinition.Builder builder = StandardTableDefinition.builder(); + builder.schema(Schema.of( + Field.of("stringField", Type.string()), + Field.of("booleanField", Type.bool()))); + table = bq.create(TableInfo.of(TABLE_ID, builder.build())); + tableSnippets = new TableSnippets(table); + } + + @AfterClass + public static void afterClass() { + bq.delete(TABLE_ID); + } + + @Test + public void testInsert() { + tableSnippets.insert("row1", "row2"); + } +} From a47e9b331ec9465fed1bf96f52d40f082e09cbd9 Mon Sep 17 00:00:00 2001 From: Kevin Deus Date: Fri, 16 Sep 2016 11:51:39 -0700 Subject: [PATCH 02/28] Starting work on copy Table ID snippet --- .../bigquery/snippets/TableSnippets.java | 21 ++++++++++++++++++- .../bigquery/snippets/ITTableSnippets.java | 6 ++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java index c7611246766d..9dadc9590218 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java @@ -17,13 +17,18 @@ package com.google.cloud.examples.bigquery.snippets; import com.google.cloud.Page; +import com.google.cloud.bigquery.BigQuery.JobField; +import com.google.cloud.bigquery.BigQuery.JobOption; import com.google.cloud.bigquery.BigQuery.TableDataListOption; +import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.CopyJobConfiguration; import com.google.cloud.bigquery.FieldValue; import com.google.cloud.bigquery.InsertAllRequest.RowToInsert; import com.google.cloud.bigquery.InsertAllResponse; import com.google.cloud.bigquery.Job; +import com.google.cloud.bigquery.JobInfo; import com.google.cloud.bigquery.Table; - +import com.google.cloud.bigquery.TableId; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -112,4 +117,18 @@ public Job copy(String datasetName, String tableName) { // [END copy] return job; } + + /** + * Example copying a table to a destination table referenced by table ID. + */ + // [TARGET copy(TableId, JobOption...)] + public Job copy(TableId destinationTable) throws BigQueryException { + // [START copy-tableid] + // Only copy the Status and User Email fields. + JobOption options = JobOption.fields(JobField.STATUS, JobField.USER_EMAIL); + Job job = table.copy(destinationTable, options); + // do something with job + // [END copy-tableid] + return job; + } } \ No newline at end of file diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java index eb341ee56072..311f126a2a76 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java @@ -20,6 +20,7 @@ import com.google.cloud.bigquery.BigQueryOptions; import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.Field.Type; +import com.google.cloud.bigquery.Job; import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardTableDefinition; import com.google.cloud.bigquery.Table; @@ -65,4 +66,9 @@ public static void afterClass() { public void testInsert() { tableSnippets.insert("row1", "row2"); } + + @Test + public void testCopyTableId() { + Job job = tableSnippets.copy(TABLE_ID); + } } From 75087649f6d8cbc8f61069ba7d96041c385e51c5 Mon Sep 17 00:00:00 2001 From: Daniel Tang Date: Fri, 16 Sep 2016 11:56:08 -0700 Subject: [PATCH 03/28] fix test setup/teardown --- .../bigquery/snippets/ITTableSnippets.java | 46 +++++++++++++++---- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java index eb341ee56072..c66b9aea9f59 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java @@ -16,10 +16,17 @@ package com.google.cloud.examples.bigquery.snippets; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import com.google.cloud.Page; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.DatasetInfo; import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.Field.Type; +import com.google.cloud.bigquery.FieldValue; +import com.google.cloud.bigquery.InsertAllResponse; import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardTableDefinition; import com.google.cloud.bigquery.Table; @@ -30,39 +37,62 @@ import org.junit.BeforeClass; import org.junit.Test; +import java.util.ArrayList; +import java.util.List; import java.util.logging.Logger; /** * Integration tests for {@link TableSnippets}. */ public class ITTableSnippets { - private static final String TABLE_NAME = "table"; - private static final String DATASET_NAME = "dataset"; - public static final TableId TABLE_ID = TableId.of(DATASET_NAME, TABLE_NAME); + private static final String TABLE_NAME = "my_table"; + private static final String DATASET_NAME = "my_dataset"; + private static final Value ROW1 = new Value("value1", true); + private static final Value ROW2 = new Value("value2", false); + private static final TableId TABLE_ID = TableId.of(DATASET_NAME, TABLE_NAME); private static final Logger log = Logger.getLogger(ITTableSnippets.class.getName()); - private static BigQuery bq; + private static BigQuery bigquery; private static Table table; private static TableSnippets tableSnippets; @BeforeClass public static void beforeClass() { - bq = BigQueryOptions.defaultInstance().service(); + bigquery = BigQueryOptions.defaultInstance().service(); + bigquery.create(DatasetInfo.builder(DATASET_NAME).build()); StandardTableDefinition.Builder builder = StandardTableDefinition.builder(); builder.schema(Schema.of( Field.of("stringField", Type.string()), Field.of("booleanField", Type.bool()))); - table = bq.create(TableInfo.of(TABLE_ID, builder.build())); + table = bigquery.create(TableInfo.of(TABLE_ID, builder.build())); tableSnippets = new TableSnippets(table); } @AfterClass public static void afterClass() { - bq.delete(TABLE_ID); + bigquery.delete(TABLE_ID); + bigquery.delete(DATASET_NAME); } @Test public void testInsert() { - tableSnippets.insert("row1", "row2"); + InsertAllResponse response = tableSnippets.insert("row1", "row2"); + assertFalse(response.hasErrors()); + Page> page = table.list(); + List> results = new ArrayList<>(); + for (List row : page.values()) { + results.add(row); + } + assertEquals(2, results.size()); + } + + private static class Value { + final String stringField; + final boolean booleanField; + + Value(String stringField, boolean booleanField) { + this.stringField = stringField; + this.booleanField = booleanField; + } } } From ff3133d59d63ae8229e13265b8dda1d684a4f7cc Mon Sep 17 00:00:00 2001 From: Kevin Deus Date: Fri, 16 Sep 2016 11:56:52 -0700 Subject: [PATCH 04/28] Still working toward a copy tableID test --- .../cloud/examples/bigquery/snippets/ITTableSnippets.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java index 311f126a2a76..47650277dcd0 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java @@ -37,7 +37,7 @@ * Integration tests for {@link TableSnippets}. */ public class ITTableSnippets { - private static final String TABLE_NAME = "table"; + private static final String TABLE_NAME = "snippetTestTable"; private static final String DATASET_NAME = "dataset"; public static final TableId TABLE_ID = TableId.of(DATASET_NAME, TABLE_NAME); private static final Logger log = Logger.getLogger(ITTableSnippets.class.getName()); From 56b385f619c8f218bd0f71e24f30779596904703 Mon Sep 17 00:00:00 2001 From: Kevin Deus Date: Fri, 16 Sep 2016 12:01:15 -0700 Subject: [PATCH 05/28] Fixed the other merge conflict. --- .../cloud/examples/bigquery/snippets/ITTableSnippets.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java index 367e5130722b..3c9cf26824a5 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java @@ -25,12 +25,9 @@ import com.google.cloud.bigquery.DatasetInfo; import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.Field.Type; -<<<<<<< HEAD import com.google.cloud.bigquery.Job; -======= import com.google.cloud.bigquery.FieldValue; import com.google.cloud.bigquery.InsertAllResponse; ->>>>>>> master import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardTableDefinition; import com.google.cloud.bigquery.Table; From 80685765575d19ecdd35c9cde86f5bc2eb2a3e59 Mon Sep 17 00:00:00 2001 From: Kevin Deus Date: Fri, 16 Sep 2016 13:47:26 -0700 Subject: [PATCH 06/28] Working copy by table ID snippet. --- .../bigquery/snippets/TableSnippets.java | 19 +++++++++++++++++-- .../bigquery/snippets/ITTableSnippets.java | 13 ++++++++++++- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java index 9dadc9590218..ff0f62b353a9 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java @@ -17,6 +17,7 @@ package com.google.cloud.examples.bigquery.snippets; import com.google.cloud.Page; +import com.google.cloud.WaitForOption; import com.google.cloud.bigquery.BigQuery.JobField; import com.google.cloud.bigquery.BigQuery.JobOption; import com.google.cloud.bigquery.BigQuery.TableDataListOption; @@ -33,6 +34,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; /* * EDITING INSTRUCTIONS @@ -120,14 +123,26 @@ public Job copy(String datasetName, String tableName) { /** * Example copying a table to a destination table referenced by table ID. + * @throws TimeoutException + * @throws InterruptedException */ // [TARGET copy(TableId, JobOption...)] public Job copy(TableId destinationTable) throws BigQueryException { // [START copy-tableid] - // Only copy the Status and User Email fields. + // As an example, this only retrieves the Status and User Email fields in the + // job RPC. JobOption options = JobOption.fields(JobField.STATUS, JobField.USER_EMAIL); Job job = table.copy(destinationTable, options); - // do something with job + + // Wait for the job to complete. + try { + Job completedJob = job.waitFor(WaitForOption.checkEvery(1, TimeUnit.SECONDS), + WaitForOption.timeout(60, TimeUnit.SECONDS)); + if (completedJob != null && completedJob.status().error() == null) { + // Job completed successfully. + } + } catch(InterruptedException | TimeoutException e) { + } // [END copy-tableid] return job; } diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java index 3c9cf26824a5..a8dd8a4fe7bc 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java @@ -99,6 +99,17 @@ private static class Value { @Test public void testCopyTableId() { - Job job = tableSnippets.copy(TABLE_ID); + StandardTableDefinition.Builder builder = StandardTableDefinition.builder(); + builder.schema(Schema.of( + Field.of("stringField", Type.string()), + Field.of("booleanField", Type.bool()))); + TableId destinationId = TableId.of(DATASET_NAME, "copy_destination"); + Table destinationTable = bigquery.create(TableInfo.of(destinationId, builder.build())); + + try { + Job job = tableSnippets.copy(destinationId); + } finally { + bigquery.delete(destinationId); + } } } From 866cf438e08e72ee0fb62b42ec8491074a5ad9e5 Mon Sep 17 00:00:00 2001 From: Kevin Deus Date: Fri, 16 Sep 2016 14:08:31 -0700 Subject: [PATCH 07/28] Moved parameters. Better snippet. --- .../examples/bigquery/snippets/TableSnippets.java | 10 ++++++---- .../examples/bigquery/snippets/ITTableSnippets.java | 5 +++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java index ff0f62b353a9..5ed8f25dc431 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java @@ -127,12 +127,14 @@ public Job copy(String datasetName, String tableName) { * @throws InterruptedException */ // [TARGET copy(TableId, JobOption...)] - public Job copy(TableId destinationTable) throws BigQueryException { + // [VARIABLE "my_dataset"] + // [VARIABLE "copy_destination"] + public Job copyTableId(String dataset, String tableName) throws BigQueryException { // [START copy-tableid] - // As an example, this only retrieves the Status and User Email fields in the - // job RPC. + TableId destinationId = TableId.of(dataset, tableName); JobOption options = JobOption.fields(JobField.STATUS, JobField.USER_EMAIL); - Job job = table.copy(destinationTable, options); + + Job job = table.copy(destinationId, options); // Wait for the job to complete. try { diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java index a8dd8a4fe7bc..4f2fa219b39a 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java @@ -99,15 +99,16 @@ private static class Value { @Test public void testCopyTableId() { + final String destinationTableName = "copy_destination"; StandardTableDefinition.Builder builder = StandardTableDefinition.builder(); builder.schema(Schema.of( Field.of("stringField", Type.string()), Field.of("booleanField", Type.bool()))); - TableId destinationId = TableId.of(DATASET_NAME, "copy_destination"); + TableId destinationId = TableId.of(DATASET_NAME, destinationTableName); Table destinationTable = bigquery.create(TableInfo.of(destinationId, builder.build())); try { - Job job = tableSnippets.copy(destinationId); + Job job = tableSnippets.copyTableId(DATASET_NAME, destinationTableName); } finally { bigquery.delete(destinationId); } From 60bc8fa67bb9d78c53e5bb359a88c1f50c0bf128 Mon Sep 17 00:00:00 2001 From: Kevin Deus Date: Fri, 16 Sep 2016 14:10:24 -0700 Subject: [PATCH 08/28] Moved parameters. Better snippet. --- .../cloud/examples/bigquery/snippets/TableSnippets.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java index 5ed8f25dc431..3bfca510879a 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java @@ -130,7 +130,7 @@ public Job copy(String datasetName, String tableName) { // [VARIABLE "my_dataset"] // [VARIABLE "copy_destination"] public Job copyTableId(String dataset, String tableName) throws BigQueryException { - // [START copy-tableid] + // [START copyTableId] TableId destinationId = TableId.of(dataset, tableName); JobOption options = JobOption.fields(JobField.STATUS, JobField.USER_EMAIL); @@ -142,10 +142,13 @@ public Job copyTableId(String dataset, String tableName) throws BigQueryExceptio WaitForOption.timeout(60, TimeUnit.SECONDS)); if (completedJob != null && completedJob.status().error() == null) { // Job completed successfully. + } else { + // Handle error case. } } catch(InterruptedException | TimeoutException e) { + // Handle interrupted wait. } - // [END copy-tableid] + // [END copyTableId] return job; } } \ No newline at end of file From 10fdfac54ea47a4e0fde710322ad5a4d8934e98d Mon Sep 17 00:00:00 2001 From: Daniel Tang Date: Fri, 16 Sep 2016 14:14:23 -0700 Subject: [PATCH 09/28] add working first TableSnippets test --- .../bigquery/snippets/ITTableSnippets.java | 74 ++++++++++++++----- 1 file changed, 56 insertions(+), 18 deletions(-) diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java index c66b9aea9f59..71505c563d89 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java @@ -19,7 +19,14 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import com.google.cloud.Page; +import java.util.List; +import java.util.Set; +import java.util.logging.Logger; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryOptions; import com.google.cloud.bigquery.DatasetInfo; @@ -32,14 +39,11 @@ import com.google.cloud.bigquery.Table; import com.google.cloud.bigquery.TableId; import com.google.cloud.bigquery.TableInfo; - -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Logger; +import com.google.common.base.Function; +import com.google.common.base.Objects; +import com.google.common.collect.FluentIterable; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; /** * Integration tests for {@link TableSnippets}. @@ -47,43 +51,58 @@ public class ITTableSnippets { private static final String TABLE_NAME = "my_table"; private static final String DATASET_NAME = "my_dataset"; + private static final String COPY_DATASET_NAME = "my_copy_dataset"; private static final Value ROW1 = new Value("value1", true); private static final Value ROW2 = new Value("value2", false); private static final TableId TABLE_ID = TableId.of(DATASET_NAME, TABLE_NAME); + private static final TableId COPY_TABLE_ID = TableId.of(COPY_DATASET_NAME, TABLE_NAME); private static final Logger log = Logger.getLogger(ITTableSnippets.class.getName()); private static BigQuery bigquery; private static Table table; + private static Table copyTable; private static TableSnippets tableSnippets; @BeforeClass public static void beforeClass() { bigquery = BigQueryOptions.defaultInstance().service(); bigquery.create(DatasetInfo.builder(DATASET_NAME).build()); + bigquery.create(DatasetInfo.builder(COPY_DATASET_NAME).build()); StandardTableDefinition.Builder builder = StandardTableDefinition.builder(); - builder.schema(Schema.of( - Field.of("stringField", Type.string()), - Field.of("booleanField", Type.bool()))); + builder.schema( + Schema.of(Field.of("stringField", Type.string()), Field.of("booleanField", Type.bool()))); table = bigquery.create(TableInfo.of(TABLE_ID, builder.build())); + copyTable = bigquery.create(TableInfo.of(COPY_TABLE_ID, builder.build())); tableSnippets = new TableSnippets(table); } @AfterClass public static void afterClass() { bigquery.delete(TABLE_ID); + bigquery.delete(COPY_TABLE_ID); bigquery.delete(DATASET_NAME); + bigquery.delete(COPY_DATASET_NAME); } @Test - public void testInsert() { + public void testInsert() throws InterruptedException { InsertAllResponse response = tableSnippets.insert("row1", "row2"); assertFalse(response.hasErrors()); - Page> page = table.list(); - List> results = new ArrayList<>(); - for (List row : page.values()) { - results.add(row); + List> rows = ImmutableList.copyOf(tableSnippets.list().values()); + while (rows.size() != 2) { + Thread.sleep(500); + rows = ImmutableList.copyOf(tableSnippets.list().values()); } - assertEquals(2, results.size()); + Set values = + FluentIterable.from(rows).transform(new Function, Value>() { + @Override + public Value apply(List row) { + return new Value(row.get(0).stringValue(), row.get(1).booleanValue()); + } + }).toSet(); + assertEquals(new Value("value1", true), new Value("value1", true)); + assertEquals(new Value("value2", false), new Value("value2", false)); + assertEquals(ImmutableSet.of(ROW2, ROW1), values); } private static class Value { @@ -94,5 +113,24 @@ private static class Value { this.stringField = stringField; this.booleanField = booleanField; } + + @Override + public boolean equals(Object obj) { + if (obj instanceof Value) { + Value o = (Value) obj; + return Objects.equal(stringField, o.stringField) && booleanField == o.booleanField; + } + return false; + } + + @Override + public int hashCode() { + return Objects.hashCode(stringField, booleanField); + } + + @Override + public String toString() { + return ""; + } } } From 1faa1e39f80f6e6969e0fca12204f838a9184251 Mon Sep 17 00:00:00 2001 From: Kevin Deus Date: Fri, 16 Sep 2016 14:19:44 -0700 Subject: [PATCH 10/28] Added todo, did some cleanup --- .../google/cloud/examples/bigquery/snippets/TableSnippets.java | 2 -- .../cloud/examples/bigquery/snippets/ITTableSnippets.java | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java index 3bfca510879a..cf0bed5b5455 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java @@ -123,8 +123,6 @@ public Job copy(String datasetName, String tableName) { /** * Example copying a table to a destination table referenced by table ID. - * @throws TimeoutException - * @throws InterruptedException */ // [TARGET copy(TableId, JobOption...)] // [VARIABLE "my_dataset"] diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java index 4f2fa219b39a..3b0f2dd6b3b9 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java @@ -109,6 +109,8 @@ public void testCopyTableId() { try { Job job = tableSnippets.copyTableId(DATASET_NAME, destinationTableName); + + // TODO: Test that the destination table has the same contents as the source. } finally { bigquery.delete(destinationId); } From fddd00e5a155e0376f7d5ab775ce02ad2b856152 Mon Sep 17 00:00:00 2001 From: Daniel Tang Date: Fri, 16 Sep 2016 14:43:55 -0700 Subject: [PATCH 11/28] change Before/AfterClass to Before/After --- .../bigquery/snippets/ITTableSnippets.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java index 71505c563d89..447c40e01d47 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java @@ -23,8 +23,8 @@ import java.util.Set; import java.util.logging.Logger; -import org.junit.AfterClass; -import org.junit.BeforeClass; +import org.junit.After; +import org.junit.Before; import org.junit.Test; import com.google.cloud.bigquery.BigQuery; @@ -58,13 +58,13 @@ public class ITTableSnippets { private static final TableId COPY_TABLE_ID = TableId.of(COPY_DATASET_NAME, TABLE_NAME); private static final Logger log = Logger.getLogger(ITTableSnippets.class.getName()); - private static BigQuery bigquery; - private static Table table; - private static Table copyTable; - private static TableSnippets tableSnippets; + private BigQuery bigquery; + private Table table; + private Table copyTable; + private TableSnippets tableSnippets; - @BeforeClass - public static void beforeClass() { + @Before + public void before() { bigquery = BigQueryOptions.defaultInstance().service(); bigquery.create(DatasetInfo.builder(DATASET_NAME).build()); bigquery.create(DatasetInfo.builder(COPY_DATASET_NAME).build()); @@ -76,8 +76,8 @@ public static void beforeClass() { tableSnippets = new TableSnippets(table); } - @AfterClass - public static void afterClass() { + @After + public void after() { bigquery.delete(TABLE_ID); bigquery.delete(COPY_TABLE_ID); bigquery.delete(DATASET_NAME); From 0efb3c129fb119855e02d3f3e98c9d25789f55f2 Mon Sep 17 00:00:00 2001 From: Kevin Deus Date: Fri, 16 Sep 2016 15:08:48 -0700 Subject: [PATCH 12/28] Made table names unique, added sample data to copy table. --- .../bigquery/snippets/ITTableSnippets.java | 127 +++++++++++++----- 1 file changed, 96 insertions(+), 31 deletions(-) diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java index 526cdcf5ac7d..189246bf2f00 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java @@ -18,20 +18,25 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.logging.Logger; -import org.junit.AfterClass; -import org.junit.BeforeClass; +import org.junit.After; +import org.junit.Before; import org.junit.Test; - +import com.google.cloud.Page; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryOptions; import com.google.cloud.bigquery.DatasetInfo; import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.Field.Type; +import com.google.cloud.bigquery.InsertAllRequest.RowToInsert; import com.google.cloud.bigquery.Job; import com.google.cloud.bigquery.FieldValue; import com.google.cloud.bigquery.InsertAllResponse; @@ -40,6 +45,7 @@ import com.google.cloud.bigquery.Table; import com.google.cloud.bigquery.TableId; import com.google.cloud.bigquery.TableInfo; +import com.google.cloud.bigquery.BigQuery.TableDataListOption; import com.google.common.base.Function; import com.google.common.base.Objects; import com.google.common.collect.FluentIterable; @@ -50,36 +56,47 @@ * Integration tests for {@link TableSnippets}. */ public class ITTableSnippets { - private static final String TABLE_NAME = "my_table"; + private static final String BASE_TABLE_NAME = "my_table"; private static final String DATASET_NAME = "my_dataset"; private static final String COPY_DATASET_NAME = "my_copy_dataset"; private static final Value ROW1 = new Value("value1", true); private static final Value ROW2 = new Value("value2", false); - private static final TableId TABLE_ID = TableId.of(DATASET_NAME, TABLE_NAME); - private static final TableId COPY_TABLE_ID = TableId.of(COPY_DATASET_NAME, TABLE_NAME); + // TODO: Make the copy table name unique. + private static final TableId COPY_TABLE_ID = TableId.of(COPY_DATASET_NAME, BASE_TABLE_NAME); private static final Logger log = Logger.getLogger(ITTableSnippets.class.getName()); - private static Table table; - private static Table copyTable; - private static TableSnippets tableSnippets; - private static BigQuery bigquery; + private Table table; + private Table copyTable; + private TableSnippets tableSnippets; + private BigQuery bigquery; + + private static int nextTableNumber = 0; + + private String GetTableName() { + return BASE_TABLE_NAME + nextTableNumber; + } + + private TableId GetTableId() { + return TableId.of(DATASET_NAME, GetTableName()); + } - @BeforeClass - public static void beforeClass() { + @Before + public void before() { + ++nextTableNumber; bigquery = BigQueryOptions.defaultInstance().service(); bigquery.create(DatasetInfo.builder(DATASET_NAME).build()); bigquery.create(DatasetInfo.builder(COPY_DATASET_NAME).build()); StandardTableDefinition.Builder builder = StandardTableDefinition.builder(); builder.schema( Schema.of(Field.of("stringField", Type.string()), Field.of("booleanField", Type.bool()))); - table = bigquery.create(TableInfo.of(TABLE_ID, builder.build())); + table = bigquery.create(TableInfo.of(GetTableId(), builder.build())); copyTable = bigquery.create(TableInfo.of(COPY_TABLE_ID, builder.build())); tableSnippets = new TableSnippets(table); } - @AfterClass - public static void afterClass() { - bigquery.delete(TABLE_ID); + @After + public void after() { + bigquery.delete(GetTableId()); bigquery.delete(COPY_TABLE_ID); bigquery.delete(DATASET_NAME); bigquery.delete(COPY_DATASET_NAME); @@ -89,6 +106,8 @@ public static void afterClass() { public void testInsert() throws InterruptedException { InsertAllResponse response = tableSnippets.insert("row1", "row2"); assertFalse(response.hasErrors()); + + // TODO: Replace the following with VerifyTestRows(table)? List> rows = ImmutableList.copyOf(tableSnippets.list().values()); while (rows.size() != 2) { Thread.sleep(500); @@ -135,22 +154,68 @@ public String toString() { } } + /** + * Inserts some data into the test table. + */ + private void InsertTestRows() { + log.info("Inserting test rows..."); + String rowId1 = "row1"; + String rowId2 = "row2"; + List rows = new ArrayList<>(); + Map row1 = new HashMap<>(); + row1.put("stringField", ROW1.stringField); + row1.put("booleanField", ROW1.booleanField); + Map row2 = new HashMap<>(); + row2.put("stringField", ROW2.stringField); + row2.put("booleanField", ROW2.booleanField); + rows.add(RowToInsert.of(rowId1, row1)); + rows.add(RowToInsert.of(rowId2, row2)); + table.insert(rows); + } + + /** + * Verifies that the given table has the rows inserted by InsertTestRows(). + * @param checkTable The table to query. + */ + private void VerifyTestRows(Table checkTable) { + List> rows = WaitForTableRows(checkTable, 2); + // Verify that the table data matches what it's supposed to. + Set values = + FluentIterable.from(rows).transform(new Function, Value>() { + @Override + public Value apply(List row) { + return new Value(row.get(0).stringValue(), row.get(1).booleanValue()); + } + }).toSet(); + assertEquals(ImmutableSet.of(ROW2, ROW1), values); + } + + private List> WaitForTableRows(Table checkTable, int numRows) { + // Wait for the data to appear. + Page> page = checkTable.list(TableDataListOption.pageSize(100)); + List> rows = ImmutableList.copyOf(page.values()); + int numSleeps = 0; + while (rows.size() != numRows) { + assertTrue(numSleeps < 10); + log.info("Sleeping and waiting for " + numRows + " test rows to appear (currently " + + rows.size() + ")..."); + try { + ++numSleeps; + Thread.sleep(5000); + } catch(InterruptedException e) { + } + page = checkTable.list(TableDataListOption.pageSize(100)); + rows = ImmutableList.copyOf(page.values()); + } + return rows; + } + @Test public void testCopyTableId() { - final String destinationTableName = "copy_destination"; - StandardTableDefinition.Builder builder = StandardTableDefinition.builder(); - builder.schema(Schema.of( - Field.of("stringField", Type.string()), - Field.of("booleanField", Type.bool()))); - TableId destinationId = TableId.of(DATASET_NAME, destinationTableName); - Table destinationTable = bigquery.create(TableInfo.of(destinationId, builder.build())); - - try { - Job job = tableSnippets.copyTableId(DATASET_NAME, destinationTableName); - - // TODO: Test that the destination table has the same contents as the source. - } finally { - bigquery.delete(destinationId); - } + InsertTestRows(); + VerifyTestRows(table); + log.info("Found test rows in original table."); + + tableSnippets.copyTableId(COPY_DATASET_NAME, BASE_TABLE_NAME); } } From b25e3ca0f5e8e4bfeebd72513fe28cf5263ed4b2 Mon Sep 17 00:00:00 2001 From: Kevin Deus Date: Fri, 16 Sep 2016 16:04:20 -0700 Subject: [PATCH 13/28] Unique copy table names, and helper functions. --- .../bigquery/snippets/ITTableSnippets.java | 70 +++++++++---------- 1 file changed, 33 insertions(+), 37 deletions(-) diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java index 1c07d5df2d5b..299f99208745 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java @@ -61,8 +61,6 @@ public class ITTableSnippets { private static final String COPY_DATASET_NAME = "my_copy_dataset"; private static final Value ROW1 = new Value("value1", true); private static final Value ROW2 = new Value("value2", false); - // TODO: Make the copy table name unique. - private static final TableId COPY_TABLE_ID = TableId.of(COPY_DATASET_NAME, BASE_TABLE_NAME); private static final Logger log = Logger.getLogger(ITTableSnippets.class.getName()); private BigQuery bigquery; @@ -81,48 +79,40 @@ public void before() { StandardTableDefinition.Builder builder = StandardTableDefinition.builder(); builder.schema( Schema.of(Field.of("stringField", Type.string()), Field.of("booleanField", Type.bool()))); - table = bigquery.create(TableInfo.of(GetTableId(), builder.build())); - copyTable = bigquery.create(TableInfo.of(COPY_TABLE_ID, builder.build())); + table = bigquery.create(TableInfo.of(getTableId(), builder.build())); + copyTable = bigquery.create(TableInfo.of(getCopyTableId(), builder.build())); tableSnippets = new TableSnippets(table); } @After public void after() { - bigquery.delete(GetTableId()); - bigquery.delete(COPY_TABLE_ID); + bigquery.delete(getTableId()); + bigquery.delete(getCopyTableId()); bigquery.delete(DATASET_NAME); bigquery.delete(COPY_DATASET_NAME); } - private String GetTableName() { + private String getTableName() { return BASE_TABLE_NAME + nextTableNumber; } - private TableId GetTableId() { - return TableId.of(DATASET_NAME, GetTableName()); + private TableId getTableId() { + return TableId.of(DATASET_NAME, getTableName()); + } + + private String getCopyTableName() { + return BASE_TABLE_NAME + "_copy_" + nextTableNumber; + } + + private TableId getCopyTableId() { + return TableId.of(COPY_DATASET_NAME, getCopyTableName()); } @Test public void testInsert() throws InterruptedException { InsertAllResponse response = tableSnippets.insert("row1", "row2"); assertFalse(response.hasErrors()); - - // TODO: Replace the following with VerifyTestRows(table)? - List> rows = ImmutableList.copyOf(tableSnippets.list().values()); - while (rows.size() != 2) { - Thread.sleep(500); - rows = ImmutableList.copyOf(tableSnippets.list().values()); - } - Set values = - FluentIterable.from(rows).transform(new Function, Value>() { - @Override - public Value apply(List row) { - return new Value(row.get(0).stringValue(), row.get(1).booleanValue()); - } - }).toSet(); - assertEquals(new Value("value1", true), new Value("value1", true)); - assertEquals(new Value("value2", false), new Value("value2", false)); - assertEquals(ImmutableSet.of(ROW2, ROW1), values); + verifyTestRows(table); } private static class Value { @@ -157,8 +147,7 @@ public String toString() { /** * Inserts some data into the test table. */ - private void InsertTestRows() { - log.info("Inserting test rows..."); + private void insertTestRows() { String rowId1 = "row1"; String rowId2 = "row2"; List rows = new ArrayList<>(); @@ -170,15 +159,19 @@ private void InsertTestRows() { row2.put("booleanField", ROW2.booleanField); rows.add(RowToInsert.of(rowId1, row1)); rows.add(RowToInsert.of(rowId2, row2)); - table.insert(rows); + InsertAllResponse response = table.insert(rows); + while (response.hasErrors()) { + log.info("Error inserting rows. Trying again..."); + response = table.insert(rows); + } } /** * Verifies that the given table has the rows inserted by InsertTestRows(). * @param checkTable The table to query. */ - private void VerifyTestRows(Table checkTable) { - List> rows = WaitForTableRows(checkTable, 2); + private void verifyTestRows(Table checkTable) { + List> rows = waitForTableRows(checkTable, 2); // Verify that the table data matches what it's supposed to. Set values = FluentIterable.from(rows).transform(new Function, Value>() { @@ -190,7 +183,14 @@ public Value apply(List row) { assertEquals(ImmutableSet.of(ROW2, ROW1), values); } - private List> WaitForTableRows(Table checkTable, int numRows) { + /** + * Waits for a specified number of rows to appear in the given table. This is used + * by verifyTestRows to wait for data to appear before verifying. + * @param checkTable + * @param numRows + * @return The rows from the table. + */ + private List> waitForTableRows(Table checkTable, int numRows) { // Wait for the data to appear. Page> page = checkTable.list(TableDataListOption.pageSize(100)); List> rows = ImmutableList.copyOf(page.values()); @@ -212,10 +212,6 @@ private List> WaitForTableRows(Table checkTable, int numRows) { @Test public void testCopyTableId() { - InsertTestRows(); - VerifyTestRows(table); - log.info("Found test rows in original table."); - - tableSnippets.copyTableId(COPY_DATASET_NAME, BASE_TABLE_NAME); + tableSnippets.copyTableId(COPY_DATASET_NAME, getCopyTableName()); } } From 32576ac9c0e40ef64f51e028cff59f8e849094e5 Mon Sep 17 00:00:00 2001 From: Daniel Tang Date: Fri, 16 Sep 2016 16:24:47 -0700 Subject: [PATCH 14/28] add testInsertParams, testList, and testCopy --- .../bigquery/snippets/TableSnippets.java | 17 ++++- .../bigquery/snippets/ITTableSnippets.java | 71 ++++++++++++++++--- 2 files changed, 75 insertions(+), 13 deletions(-) diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java index cf0bed5b5455..99cf03682a6a 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java @@ -82,7 +82,7 @@ public InsertAllResponse insertWithParams(String rowId1, String rowId2) { // [START insertWithParams] List rows = new ArrayList<>(); Map row1 = new HashMap<>(); - row1.put("stringField", "value1"); + row1.put("stringField", 1); row1.put("booleanField", true); Map row2 = new HashMap<>(); row2.put("stringField", "value2"); @@ -116,7 +116,20 @@ public Page> list() { public Job copy(String datasetName, String tableName) { // [START copy] Job job = table.copy(datasetName, tableName); - // do something with job + + // Wait for the job to complete. + try { + Job completedJob = job.waitFor(WaitForOption.checkEvery(1, TimeUnit.SECONDS), + WaitForOption.timeout(60, TimeUnit.SECONDS)); + if (completedJob != null && completedJob.status().error() == null) { + // Job completed successfully. + } else { + // Handle error case. + } + } catch (InterruptedException | TimeoutException e) { + // Handle interrupted wait. + } + // [END copy] return job; } diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java index 299f99208745..a0e30f40ab6a 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java @@ -28,24 +28,27 @@ import java.util.logging.Logger; import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; + import com.google.cloud.Page; import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption; +import com.google.cloud.bigquery.BigQuery.TableDataListOption; import com.google.cloud.bigquery.BigQueryOptions; import com.google.cloud.bigquery.DatasetInfo; import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.Field.Type; -import com.google.cloud.bigquery.InsertAllRequest.RowToInsert; -import com.google.cloud.bigquery.Job; import com.google.cloud.bigquery.FieldValue; +import com.google.cloud.bigquery.InsertAllRequest.RowToInsert; import com.google.cloud.bigquery.InsertAllResponse; import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardTableDefinition; import com.google.cloud.bigquery.Table; import com.google.cloud.bigquery.TableId; import com.google.cloud.bigquery.TableInfo; -import com.google.cloud.bigquery.BigQuery.TableDataListOption; import com.google.common.base.Function; import com.google.common.base.Objects; import com.google.common.collect.FluentIterable; @@ -63,24 +66,27 @@ public class ITTableSnippets { private static final Value ROW2 = new Value("value2", false); private static final Logger log = Logger.getLogger(ITTableSnippets.class.getName()); - private BigQuery bigquery; + private static BigQuery bigquery; private Table table; - private Table copyTable; private TableSnippets tableSnippets; private static int nextTableNumber = 0; - @Before - public void before() { - ++nextTableNumber; + @BeforeClass + public static void beforeClass() { bigquery = BigQueryOptions.defaultInstance().service(); bigquery.create(DatasetInfo.builder(DATASET_NAME).build()); bigquery.create(DatasetInfo.builder(COPY_DATASET_NAME).build()); + } + + @Before + public void before() { + ++nextTableNumber; StandardTableDefinition.Builder builder = StandardTableDefinition.builder(); builder.schema( Schema.of(Field.of("stringField", Type.string()), Field.of("booleanField", Type.bool()))); table = bigquery.create(TableInfo.of(getTableId(), builder.build())); - copyTable = bigquery.create(TableInfo.of(getCopyTableId(), builder.build())); + bigquery.create(TableInfo.of(getCopyTableId(), builder.build())); tableSnippets = new TableSnippets(table); } @@ -88,8 +94,12 @@ public void before() { public void after() { bigquery.delete(getTableId()); bigquery.delete(getCopyTableId()); - bigquery.delete(DATASET_NAME); - bigquery.delete(COPY_DATASET_NAME); + } + + @AfterClass + public static void afterClass() { + bigquery.delete(DATASET_NAME, DatasetDeleteOption.deleteContents()); + bigquery.delete(COPY_DATASET_NAME, DatasetDeleteOption.deleteContents()); } private String getTableName() { @@ -115,6 +125,45 @@ public void testInsert() throws InterruptedException { verifyTestRows(table); } + @Test + public void testInsertParams() throws InterruptedException { + InsertAllResponse response = tableSnippets.insertWithParams("row1", "row2"); + assertTrue(response.hasErrors()); + List> rows = ImmutableList.copyOf(tableSnippets.list().values()); + while (rows.size() == 0) { + Thread.sleep(500); + rows = ImmutableList.copyOf(tableSnippets.list().values()); + } + Set values = + FluentIterable.from(rows).transform(new Function, Value>() { + @Override + public Value apply(List row) { + return new Value(row.get(0).stringValue(), row.get(1).booleanValue()); + } + }).toSet(); + assertEquals(ImmutableSet.of(ROW2), values); + } + + @Test + public void testList() throws InterruptedException { + List> rows = ImmutableList.copyOf(tableSnippets.list().values()); + assertEquals(0, rows.size()); + + InsertAllResponse response = tableSnippets.insert("row1", "row2"); + assertFalse(response.hasErrors()); + rows = ImmutableList.copyOf(tableSnippets.list().values()); + while (rows.size() == 0) { + Thread.sleep(500); + rows = ImmutableList.copyOf(tableSnippets.list().values()); + } + assertEquals(2, rows.size()); + } + + @Test + public void testCopy() { + tableSnippets.copy(COPY_DATASET_NAME, BASE_TABLE_NAME); + } + private static class Value { final String stringField; final boolean booleanField; From c1e416a0be7bff814ce54e9ab3b555bdc33e2c06 Mon Sep 17 00:00:00 2001 From: Kevin Deus Date: Fri, 16 Sep 2016 16:35:00 -0700 Subject: [PATCH 15/28] Add testExtract, and some minor cleanup. --- .../bigquery/snippets/TableSnippets.java | 29 +++++++++++++++++++ .../bigquery/snippets/ITTableSnippets.java | 11 ++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java index cf0bed5b5455..89eefea85df2 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java @@ -149,4 +149,33 @@ public Job copyTableId(String dataset, String tableName) throws BigQueryExceptio // [END copyTableId] return job; } + + /** + * Example extracting data to Google Cloud Storage. + */ + // [TARGET extract(String, List, BigQuery.JobOption...)] + // [VARIABLE "CSV"] + // [VARIABLE "gs://bucket_name/filename.csv"] + public Job extract(String format, String gcsUrl) { + // [START extract] + List destinationUris = new ArrayList<>(); + destinationUris.add(gcsUrl); + + Job job = table.extract(format, destinationUris); + + // Wait for the job to complete. + try { + Job completedJob = job.waitFor(WaitForOption.checkEvery(1, TimeUnit.SECONDS), + WaitForOption.timeout(60, TimeUnit.SECONDS)); + if (completedJob != null && completedJob.status().error() == null) { + // Job completed successfully. + } else { + // Handle error case. + } + } catch(InterruptedException | TimeoutException e) { + // Handle interrupted wait. + } + // [END extract] + return job; + } } \ No newline at end of file diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java index 299f99208745..8db1aa8b70a1 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java @@ -109,7 +109,8 @@ private TableId getCopyTableId() { } @Test - public void testInsert() throws InterruptedException { + public void testInsert() { + log.info("testInsert"); InsertAllResponse response = tableSnippets.insert("row1", "row2"); assertFalse(response.hasErrors()); verifyTestRows(table); @@ -212,6 +213,14 @@ private List> waitForTableRows(Table checkTable, int numRows) { @Test public void testCopyTableId() { + log.info("testCopyTableId"); tableSnippets.copyTableId(COPY_DATASET_NAME, getCopyTableName()); } + + @Test + public void testExtract() { + log.info("testExtract"); + String projectId = bigquery.options().projectId(); + tableSnippets.extract("CSV", "gs://" + projectId + ".appspot.com/extractTest.csv"); + } } From 7049cd4de2ccf0dd26b5f6e5bb9f6b8b731d8df6 Mon Sep 17 00:00:00 2001 From: Kevin Deus Date: Fri, 16 Sep 2016 16:37:02 -0700 Subject: [PATCH 16/28] Remove unnecessary imports. --- .../google/cloud/examples/bigquery/snippets/TableSnippets.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java index e2f945d9775c..6ee2e7cf7f1c 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java @@ -22,12 +22,10 @@ import com.google.cloud.bigquery.BigQuery.JobOption; import com.google.cloud.bigquery.BigQuery.TableDataListOption; import com.google.cloud.bigquery.BigQueryException; -import com.google.cloud.bigquery.CopyJobConfiguration; import com.google.cloud.bigquery.FieldValue; import com.google.cloud.bigquery.InsertAllRequest.RowToInsert; import com.google.cloud.bigquery.InsertAllResponse; import com.google.cloud.bigquery.Job; -import com.google.cloud.bigquery.JobInfo; import com.google.cloud.bigquery.Table; import com.google.cloud.bigquery.TableId; import java.util.ArrayList; From 524f37b627c4c45406f3b85587ba9868ddeac150 Mon Sep 17 00:00:00 2001 From: Daniel Tang Date: Fri, 16 Sep 2016 16:44:35 -0700 Subject: [PATCH 17/28] fix TARGET parameters * type parameters shouldn't be specified * outer class names should be omitted --- .../examples/bigquery/snippets/TableSnippets.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java index 6ee2e7cf7f1c..76ada02c5b55 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java @@ -50,7 +50,7 @@ public TableSnippets(Table table) { /** * Example of inserting rows into a table. */ - // [TARGET insert(Iterable)] + // [TARGET insert(Iterable)] // [VARIABLE "rowId1"] // [VARIABLE "rowId2"] public InsertAllResponse insert(String rowId1, String rowId2) { @@ -73,7 +73,7 @@ public InsertAllResponse insert(String rowId1, String rowId2) { /** * Example of inserting rows into a table which ignores invalid rows. */ - // [TARGET insert(Iterable, boolean, boolean)] + // [TARGET insert(Iterable, boolean, boolean)] // [VARIABLE "rowId1"] // [VARIABLE "rowId2"] public InsertAllResponse insertWithParams(String rowId1, String rowId2) { @@ -96,7 +96,7 @@ public InsertAllResponse insertWithParams(String rowId1, String rowId2) { /** * Example of getting a paginated list of rows in a table. */ - // [TARGET list(BigQuery.TableDataListOption...)] + // [TARGET list(TableDataListOption...)] public Page> list() { // [START list] Page> page = table.list(TableDataListOption.pageSize(100)); @@ -108,7 +108,7 @@ public Page> list() { /** * Example of copying a table to a destination table and dataset referenced by name. */ - // [TARGET copy(String, String, BigQuery.JobOption...)] + // [TARGET copy(String, String, JobOption...)] // [VARIABLE "my_dataset"] // [VARIABLE "my_destination_table"] public Job copy(String datasetName, String tableName) { @@ -164,7 +164,7 @@ public Job copyTableId(String dataset, String tableName) throws BigQueryExceptio /** * Example extracting data to Google Cloud Storage. */ - // [TARGET extract(String, List, BigQuery.JobOption...)] + // [TARGET extract(String, List, JobOption...)] // [VARIABLE "CSV"] // [VARIABLE "gs://bucket_name/filename.csv"] public Job extract(String format, String gcsUrl) { From 63de7b4fbe053e33adcdac24a040795a9028a520 Mon Sep 17 00:00:00 2001 From: Thomas Coffee Date: Fri, 16 Sep 2016 16:46:01 -0700 Subject: [PATCH 18/28] Add tests for Table exists, reload, update, delete --- .../bigquery/snippets/TableSnippets.java | 52 +++++++++++++++++++ .../bigquery/snippets/ITTableSnippets.java | 36 ++++++++++++- 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java index 76ada02c5b55..6c2b2cd1cc94 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java @@ -20,6 +20,9 @@ import com.google.cloud.WaitForOption; import com.google.cloud.bigquery.BigQuery.JobField; import com.google.cloud.bigquery.BigQuery.JobOption; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQuery.TableOption; +import com.google.cloud.bigquery.BigQuery.TableField; import com.google.cloud.bigquery.BigQuery.TableDataListOption; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.FieldValue; @@ -28,6 +31,7 @@ import com.google.cloud.bigquery.Job; import com.google.cloud.bigquery.Table; import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.spi.BigQueryRpc; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -47,6 +51,54 @@ public TableSnippets(Table table) { this.table = table; } + /** + * Example of ensuring that a table exists. + */ + // [TARGET exists()] + public void checkExists() { + // [START checkExists] + if (!table.exists()) { + throw new RuntimeException("Table does not exist."); + } + // [END checkExists] + } + + /** + * Example of fetching a table's latest information, specifying particular table field options. + */ + // [TARGET reload(BigQuery.TableOption... options)] + // [VARIABLE TableField.LAST_MODIFIED_TIME] + // [VARIABLE TableField.NUM_ROWS] + public Table reloadTableWithFields(TableField... fields) { + // [START reloadTableWithFields] + Table reloaded = table.reload(TableOption.fields(fields)); + // [END reloadTableWithFields] + return reloaded; + } + + /** + * Example of updating a table's information, specifying particular table field options. + */ + // [TARGET update(BigQuery.TableOption... options)] + // [VARIABLE TableField.LAST_MODIFIED_TIME] + // [VARIABLE TableField.NUM_ROWS] + public Table updateTableWithFields(TableField... fields) { + // [START updateTableWithFields] + Table updated = table.update(TableOption.fields(fields)); + // [END updateTableWithFields] + return updated; + } + + /** + * Example of deleting a table. + */ + // [TARGET delete()] + public void delete() { + // [START delete] + table.delete(); + // [END delete] + } + /** * Example of inserting rows into a table. */ diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java index cbfcf171f576..1c83706e904d 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java @@ -37,6 +37,7 @@ import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption; import com.google.cloud.bigquery.BigQuery.TableDataListOption; +import com.google.cloud.bigquery.BigQuery.TableField; import com.google.cloud.bigquery.BigQueryOptions; import com.google.cloud.bigquery.DatasetInfo; import com.google.cloud.bigquery.Field; @@ -70,6 +71,13 @@ public class ITTableSnippets { private Table table; private TableSnippets tableSnippets; + private static final String DOOMED_TABLE_NAME = "doomed_table"; + private static final String DOOMED_DATASET_NAME = "doomed_dataset"; + public static final TableId DOOMED_TABLE_ID = TableId.of(DOOMED_DATASET_NAME, DOOMED_TABLE_NAME); + + private static Table doomedTable; + private static TableSnippets doomedTableSnippets; + private static int nextTableNumber = 0; @BeforeClass @@ -77,6 +85,7 @@ public static void beforeClass() { bigquery = BigQueryOptions.defaultInstance().service(); bigquery.create(DatasetInfo.builder(DATASET_NAME).build()); bigquery.create(DatasetInfo.builder(COPY_DATASET_NAME).build()); + bigquery.create(DatasetInfo.builder(DOOMED_DATASET_NAME).build()); } @Before @@ -88,18 +97,23 @@ public void before() { table = bigquery.create(TableInfo.of(getTableId(), builder.build())); bigquery.create(TableInfo.of(getCopyTableId(), builder.build())); tableSnippets = new TableSnippets(table); + + doomedTable = bigquery.create(TableInfo.of(DOOMED_TABLE_ID, builder.build())); + doomedTableSnippets = new TableSnippets(doomedTable); } @After public void after() { bigquery.delete(getTableId()); bigquery.delete(getCopyTableId()); + bigquery.delete(DOOMED_TABLE_ID); } @AfterClass public static void afterClass() { bigquery.delete(DATASET_NAME, DatasetDeleteOption.deleteContents()); bigquery.delete(COPY_DATASET_NAME, DatasetDeleteOption.deleteContents()); + bigquery.delete(DOOMED_DATASET_NAME, DatasetDeleteOption.deleteContents()); } private String getTableName() { @@ -118,9 +132,29 @@ private TableId getCopyTableId() { return TableId.of(COPY_DATASET_NAME, getCopyTableName()); } + @Test + public void testCheckExists() { + tableSnippets.checkExists(); + } + + @Test + public void testReloadTableWithFields() { + tableSnippets.reloadTableWithFields(TableField.LAST_MODIFIED_TIME, TableField.NUM_ROWS); + } + + @Test + public void testUpdateTableWithFields() { + tableSnippets.updateTableWithFields(TableField.LAST_MODIFIED_TIME, TableField.NUM_ROWS); + } + + @Test + public void testDelete() { + doomedTableSnippets.delete(); + } + @Test public void testInsert() { - log.info("testInsert"); + log.info("testInsert"); InsertAllResponse response = tableSnippets.insert("row1", "row2"); assertFalse(response.hasErrors()); verifyTestRows(table); From c5e31c67c0b9f1ac7b8360d15bc3a6cd4ee78453 Mon Sep 17 00:00:00 2001 From: Kevin Deus Date: Fri, 16 Sep 2016 16:49:45 -0700 Subject: [PATCH 19/28] Better extract multiple, and added test for extract single. --- .../bigquery/snippets/TableSnippets.java | 40 ++++++++++++++++--- .../bigquery/snippets/ITTableSnippets.java | 16 ++++++-- 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java index 6ee2e7cf7f1c..55a546497cec 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java @@ -162,15 +162,17 @@ public Job copyTableId(String dataset, String tableName) throws BigQueryExceptio } /** - * Example extracting data to Google Cloud Storage. + * Example extracting data to a list of Google Cloud Storage files. */ // [TARGET extract(String, List, BigQuery.JobOption...)] // [VARIABLE "CSV"] - // [VARIABLE "gs://bucket_name/filename.csv"] - public Job extract(String format, String gcsUrl) { - // [START extract] + // [VARIABLE "gs://bucket_name/PartitionA_*.csv"] + // [VARIABLE "gs://bucket_name/PartitionB_*.csv"] + public Job extractList(String format, String gcsUrl1, String gcsUrl2) { + // [START extractList] List destinationUris = new ArrayList<>(); - destinationUris.add(gcsUrl); + destinationUris.add(gcsUrl1); + destinationUris.add(gcsUrl2); Job job = table.extract(format, destinationUris); @@ -186,7 +188,33 @@ public Job extract(String format, String gcsUrl) { } catch(InterruptedException | TimeoutException e) { // Handle interrupted wait. } - // [END extract] + // [END extractList] + return job; + } + + /** + * Example extracting data to single Google Cloud Storage file. + */ + // [TARGET extract(String, String, BigQuery.JobOption...)] + // [VARIABLE "CSV"] + // [VARIABLE "gs://bucket_name/filename.csv"] + public Job extractSingle(String format, String gcsUrl) { + // [START extractSingle] + Job job = table.extract(format, gcsUrl); + + // Wait for the job to complete. + try { + Job completedJob = job.waitFor(WaitForOption.checkEvery(1, TimeUnit.SECONDS), + WaitForOption.timeout(60, TimeUnit.SECONDS)); + if (completedJob != null && completedJob.status().error() == null) { + // Job completed successfully. + } else { + // Handle error case. + } + } catch(InterruptedException | TimeoutException e) { + // Handle interrupted wait. + } + // [END extractSingle] return job; } } \ No newline at end of file diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java index cbfcf171f576..c24e7e93530d 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java @@ -267,9 +267,19 @@ public void testCopyTableId() { } @Test - public void testExtract() { - log.info("testExtract"); + public void testExtractList() { + log.info("testExtractList"); String projectId = bigquery.options().projectId(); - tableSnippets.extract("CSV", "gs://" + projectId + ".appspot.com/extractTest.csv"); + String gcsFile1 = "gs://" + projectId + ".appspot.com/extractTestA_*.csv"; + String gcsFile2 = "gs://" + projectId + ".appspot.com/extractTestB_*.csv"; + tableSnippets.extractList("CSV", gcsFile1, gcsFile2); + } + + @Test + public void testExtractSingle() { + log.info("testExtractSingle"); + String projectId = bigquery.options().projectId(); + String gcsFile = "gs://" + projectId + ".appspot.com/extractTest.csv"; + tableSnippets.extractSingle("CSV", gcsFile); } } From b8f6b795f819d666105a5cdfb910eb32b88ef811 Mon Sep 17 00:00:00 2001 From: Thomas Coffee Date: Fri, 16 Sep 2016 17:04:02 -0700 Subject: [PATCH 20/28] Add test logs for Table exists, reload, update, delete --- .../cloud/examples/bigquery/snippets/ITTableSnippets.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java index 1cb4bfa2cb94..5d4306918575 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java @@ -134,21 +134,25 @@ private TableId getCopyTableId() { @Test public void testCheckExists() { + log.info("testCheckExists"); tableSnippets.checkExists(); } @Test public void testReloadTableWithFields() { + log.info("testReloadTableWithFields"); tableSnippets.reloadTableWithFields(TableField.LAST_MODIFIED_TIME, TableField.NUM_ROWS); } @Test public void testUpdateTableWithFields() { + log.info("testUpdateTableWithFields"); tableSnippets.updateTableWithFields(TableField.LAST_MODIFIED_TIME, TableField.NUM_ROWS); } @Test public void testDelete() { + log.info("testDelete"); doomedTableSnippets.delete(); } From 740247c98cb2b02520a920f582a3ba60b3139864 Mon Sep 17 00:00:00 2001 From: Kevin Deus Date: Fri, 16 Sep 2016 17:06:28 -0700 Subject: [PATCH 21/28] Added loadList snippet and test. --- .../bigquery/snippets/TableSnippets.java | 31 +++++++++++++++++++ .../bigquery/snippets/ITTableSnippets.java | 9 ++++++ 2 files changed, 40 insertions(+) diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java index 3aa779c8cdd6..942b152792b8 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java @@ -26,6 +26,7 @@ import com.google.cloud.bigquery.BigQuery.TableDataListOption; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.FieldValue; +import com.google.cloud.bigquery.FormatOptions; import com.google.cloud.bigquery.InsertAllRequest.RowToInsert; import com.google.cloud.bigquery.InsertAllResponse; import com.google.cloud.bigquery.Job; @@ -269,4 +270,34 @@ public Job extractSingle(String format, String gcsUrl) { // [END extractSingle] return job; } + + /** + * Example extracting data to a list of Google Cloud Storage files. + */ + // [TARGET load(String, List, JobOption...)] + // [VARIABLE "gs://bucket_name/PartitionA_000000000000.csv"] + // [VARIABLE "gs://bucket_name/PartitionB_000000000000.csv"] + public Job loadList(String gcsUrl1, String gcsUrl2) { + // [START loadList] + List sourceUris = new ArrayList<>(); + sourceUris.add(gcsUrl1); + sourceUris.add(gcsUrl2); + + Job job = table.load(FormatOptions.csv(), sourceUris); + + // Wait for the job to complete. + try { + Job completedJob = job.waitFor(WaitForOption.checkEvery(1, TimeUnit.SECONDS), + WaitForOption.timeout(60, TimeUnit.SECONDS)); + if (completedJob != null && completedJob.status().error() == null) { + // Job completed successfully. + } else { + // Handle error case. + } + } catch(InterruptedException | TimeoutException e) { + // Handle interrupted wait. + } + // [END loadList] + return job; + } } \ No newline at end of file diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java index 1cb4bfa2cb94..de0515c1fb60 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java @@ -316,4 +316,13 @@ public void testExtractSingle() { String gcsFile = "gs://" + projectId + ".appspot.com/extractTest.csv"; tableSnippets.extractSingle("CSV", gcsFile); } + + @Test + public void testLoadList() { + log.info("testLoadList"); + String projectId = bigquery.options().projectId(); + String gcsFile1 = "gs://" + projectId + ".appspot.com/extractTestA_000000000000.csv"; + String gcsFile2 = "gs://" + projectId + ".appspot.com/extractTestB_000000000000.csv"; + tableSnippets.loadList(gcsFile1, gcsFile2); + } } From 56fd8ed24287c551dc312e31961a70007f4549e9 Mon Sep 17 00:00:00 2001 From: Kevin Deus Date: Fri, 16 Sep 2016 17:08:20 -0700 Subject: [PATCH 22/28] Load test creates files first. --- .../examples/bigquery/snippets/ITTableSnippets.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java index de0515c1fb60..5651410a69a6 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java @@ -321,8 +321,13 @@ public void testExtractSingle() { public void testLoadList() { log.info("testLoadList"); String projectId = bigquery.options().projectId(); - String gcsFile1 = "gs://" + projectId + ".appspot.com/extractTestA_000000000000.csv"; - String gcsFile2 = "gs://" + projectId + ".appspot.com/extractTestB_000000000000.csv"; + String gcsFile1 = "gs://" + projectId + ".appspot.com/loadTest1.csv"; + String gcsFile2 = "gs://" + projectId + ".appspot.com/loadTest2.csv"; + + // Before we can load, we should make sure those files exist. + tableSnippets.extractSingle("CSV", gcsFile1); + tableSnippets.extractSingle("CSV", gcsFile2); + tableSnippets.loadList(gcsFile1, gcsFile2); } } From ee12c01c1f8a79820037c6b6272d6944d8f77c53 Mon Sep 17 00:00:00 2001 From: Kevin Deus Date: Fri, 16 Sep 2016 17:25:44 -0700 Subject: [PATCH 23/28] Added loadSing.e --- .../bigquery/snippets/TableSnippets.java | 39 +++++++++++++++---- .../bigquery/snippets/ITTableSnippets.java | 12 ++++++ 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java index 942b152792b8..4d3162c690e1 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java @@ -219,8 +219,8 @@ public Job copyTableId(String dataset, String tableName) throws BigQueryExceptio */ // [TARGET extract(String, List, JobOption...)] // [VARIABLE "CSV"] - // [VARIABLE "gs://bucket_name/PartitionA_*.csv"] - // [VARIABLE "gs://bucket_name/PartitionB_*.csv"] + // [VARIABLE "gs://myapp.appspot.com/PartitionA_*.csv"] + // [VARIABLE "gs://myapp.appspot.com/PartitionB_*.csv"] public Job extractList(String format, String gcsUrl1, String gcsUrl2) { // [START extractList] List destinationUris = new ArrayList<>(); @@ -248,9 +248,9 @@ public Job extractList(String format, String gcsUrl1, String gcsUrl2) { /** * Example extracting data to single Google Cloud Storage file. */ - // [TARGET extract(String, String, BigQuery.JobOption...)] + // [TARGET extract(String, String, JobOption...)] // [VARIABLE "CSV"] - // [VARIABLE "gs://bucket_name/filename.csv"] + // [VARIABLE "gs://myapp.appspot.com/filename.csv"] public Job extractSingle(String format, String gcsUrl) { // [START extractSingle] Job job = table.extract(format, gcsUrl); @@ -272,11 +272,11 @@ public Job extractSingle(String format, String gcsUrl) { } /** - * Example extracting data to a list of Google Cloud Storage files. + * Example loading data from a list of Google Cloud Storage files. */ // [TARGET load(String, List, JobOption...)] - // [VARIABLE "gs://bucket_name/PartitionA_000000000000.csv"] - // [VARIABLE "gs://bucket_name/PartitionB_000000000000.csv"] + // [VARIABLE "gs://myapp.appspot.com/PartitionA_000000000000.csv"] + // [VARIABLE "gs://myapp.appspot.com/PartitionB_000000000000.csv"] public Job loadList(String gcsUrl1, String gcsUrl2) { // [START loadList] List sourceUris = new ArrayList<>(); @@ -300,4 +300,29 @@ public Job loadList(String gcsUrl1, String gcsUrl2) { // [END loadList] return job; } + + /** + * Example loading data from a single Google Cloud Storage file. + */ + // [TARGET load(String, String, JobOption...)] + // [VARIABLE "gs://myapp.appspot.com/filename.csv"] + public Job loadSingle(String sourceUri) { + // [START loadSingle] + Job job = table.load(FormatOptions.csv(), sourceUri); + + // Wait for the job to complete. + try { + Job completedJob = job.waitFor(WaitForOption.checkEvery(1, TimeUnit.SECONDS), + WaitForOption.timeout(60, TimeUnit.SECONDS)); + if (completedJob != null && completedJob.status().error() == null) { + // Job completed successfully. + } else { + // Handle error case. + } + } catch(InterruptedException | TimeoutException e) { + // Handle interrupted wait. + } + // [END loadSingle] + return job; + } } \ No newline at end of file diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java index 13564c66b4b4..6e84ad8e00b1 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java @@ -334,4 +334,16 @@ public void testLoadList() { tableSnippets.loadList(gcsFile1, gcsFile2); } + + @Test + public void testLoadSingle() { + log.info("testLoadSingle"); + String projectId = bigquery.options().projectId(); + String gcsFile = "gs://" + projectId + ".appspot.com/loadSingle.csv"; + + // Before we can load, we should make sure the file exists. + tableSnippets.extractSingle("CSV", gcsFile); + + tableSnippets.loadSingle(gcsFile); + } } From 1c03b0aefd0e03d647954177e642df2219fdbce8 Mon Sep 17 00:00:00 2001 From: Thomas Coffee Date: Fri, 16 Sep 2016 17:36:51 -0700 Subject: [PATCH 24/28] Add docs for Table exists, reload, update, delete --- .../java/com/google/cloud/bigquery/Table.java | 155 ++++++++++++++++++ .../bigquery/snippets/TableSnippets.java | 12 +- 2 files changed, 161 insertions(+), 6 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Table.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Table.java index d76692635508..0568a1ae8e16 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Table.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Table.java @@ -139,6 +139,13 @@ public Table build() { /** * Checks if this table exists. * + *

Example of ensuring that a table exists. + * 

 {@code
+   * if (!table.exists()) {
+   *  throw new RuntimeException("Table does not exist.");
+   * }
+   * }
+ * * @return {@code true} if this table exists, {@code false} otherwise * @throws BigQueryException upon failure */ @@ -149,6 +156,13 @@ public boolean exists() { /** * Fetches current table's latest information. Returns {@code null} if the table does not exist. * + *

Example of fetching a table's latest information, specifying particular table field options. + * 

 {@code
+   * TableField field1 = TableField.LAST_MODIFIED_TIME;
+   * TableField field2 = TableField.NUM_ROWS;
+   * Table reloaded = table.reload(TableOption.fields(field1, field2));
+   * }
+ * * @param options table options * @return a {@code Table} object with latest information or {@code null} if not found * @throws BigQueryException upon failure @@ -161,6 +175,13 @@ public Table reload(TableOption... options) { * Updates the table's information with this table's information. Dataset's and table's * user-defined ids cannot be changed. A new {@code Table} object is returned. * + *

Example of updating a table's information, specifying particular table field options. + * 

 {@code
+   * TableField field1 = TableField.LAST_MODIFIED_TIME;
+   * TableField field2 = TableField.NUM_ROWS;
+   * Table updated = table.update(TableOption.fields(field1, field2));
+   * }
+ * * @param options dataset options * @return a {@code Table} object with updated information * @throws BigQueryException upon failure @@ -172,6 +193,11 @@ public Table update(TableOption... options) { /** * Deletes this table. * + *

Example of deleting a table. + * 

 {@code
+   * table.delete();
+   * }
+ * * @return {@code true} if table was deleted, {@code false} if it was not found * @throws BigQueryException upon failure */ @@ -182,6 +208,23 @@ public boolean delete() { /** * Insert rows into the table. * + *

Example of inserting rows into a table. + * 

 {@code
+   * String rowId1 = "rowId1";
+   * String rowId2 = "rowId2";
+   * List rows = new ArrayList<>();
+   * Map row1 = new HashMap<>();
+   * row1.put("stringField", "value1");
+   * row1.put("booleanField", true);
+   * Map row2 = new HashMap<>();
+   * row2.put("stringField", "value2");
+   * row2.put("booleanField", false);
+   * rows.add(RowToInsert.of(rowId1, row1));
+   * rows.add(RowToInsert.of(rowId2, row2));
+   * InsertAllResponse response = table.insert(rows);
+   * // do something with response
+   * }
+ * * @param rows rows to be inserted * @throws BigQueryException upon failure */ @@ -193,6 +236,23 @@ public InsertAllResponse insert(Iterable rows) /** * Insert rows into the table. * + *

Example of inserting rows into a table which ignores invalid rows. + * 

 {@code
+   * String rowId1 = "rowId1";
+   * String rowId2 = "rowId2";
+   * List rows = new ArrayList<>();
+   * Map row1 = new HashMap<>();
+   * row1.put("stringField", 1);
+   * row1.put("booleanField", true);
+   * Map row2 = new HashMap<>();
+   * row2.put("stringField", "value2");
+   * row2.put("booleanField", false);
+   * rows.add(RowToInsert.of(rowId1, row1));
+   * rows.add(RowToInsert.of(rowId2, row2));
+   * InsertAllResponse response = table.insert(rows, true, true);
+   * // do something with response
+   * }
+ * * @param rows rows to be inserted * @param skipInvalidRows whether to insert all valid rows, even if invalid rows exist. If not set * the entire insert operation will fail if rows to be inserted contain an invalid row @@ -213,6 +273,12 @@ public InsertAllResponse insert(Iterable rows, /** * Returns the paginated list rows in this table. * + *

Example of getting a paginated list of rows in a table. + * 

 {@code
+   * Page> page = table.list(TableDataListOption.pageSize(100));
+   * // do something with page
+   * }
+ * * @param options table data list options * @throws BigQueryException upon failure */ @@ -225,6 +291,27 @@ public Page> list(TableDataListOption... options) * Starts a BigQuery Job to copy the current table to the provided destination table. Returns the * started {@link Job} object. * + *

Example of copying a table to a destination table and dataset referenced by name. + * 

 {@code
+   * String datasetName = "my_dataset";
+   * String tableName = "my_destination_table";
+   * Job job = table.copy(datasetName, tableName);
+   * 
+   * // Wait for the job to complete.
+   * try {
+   *   Job completedJob = job.waitFor(WaitForOption.checkEvery(1, TimeUnit.SECONDS),
+   *       WaitForOption.timeout(60, TimeUnit.SECONDS));
+   *   if (completedJob != null && completedJob.status().error() == null) {
+   *     // Job completed successfully.
+   *   } else {
+   *     // Handle error case.
+   *   }
+   * } catch (InterruptedException | TimeoutException e) {
+   *   // Handle interrupted wait.
+   * }
+   * 
+   * }
+ * * @param destinationDataset the user-defined id of the destination dataset * @param destinationTable the user-defined id of the destination table * @param options job options @@ -239,6 +326,29 @@ public Job copy(String destinationDataset, String destinationTable, JobOption... * Starts a BigQuery Job to copy the current table to the provided destination table. Returns the * started {@link Job} object. * + *

Example copying a table to a destination table referenced by table ID. + * 

 {@code
+   * String dataset = "my_dataset";
+   * String tableName = "copy_destination";
+   * TableId destinationId = TableId.of(dataset, tableName);
+   * JobOption options = JobOption.fields(JobField.STATUS, JobField.USER_EMAIL);
+   * 
+   * Job job = table.copy(destinationId, options);
+   * 
+   * // Wait for the job to complete.
+   * try {
+   *   Job completedJob = job.waitFor(WaitForOption.checkEvery(1, TimeUnit.SECONDS),
+   *       WaitForOption.timeout(60, TimeUnit.SECONDS));
+   *   if (completedJob != null && completedJob.status().error() == null) {
+   *     // Job completed successfully.
+   *   } else {
+   *     // Handle error case.
+   *   }
+   * } catch(InterruptedException | TimeoutException e) {
+   *   // Handle interrupted wait.
+   * }
+   * }
+ * * @param destinationTable the destination table of the copy job * @param options job options * @throws BigQueryException upon failure @@ -253,6 +363,26 @@ public Job copy(TableId destinationTable, JobOption... options) * Starts a BigQuery Job to extract the current table to the provided destination URI. Returns the * started {@link Job} object. * + *

Example extracting data to single Google Cloud Storage file. + * 

 {@code
+   * String format = "CSV";
+   * String gcsUrl = "gs://bucket_name/filename.csv";
+   * Job job = table.extract(format, gcsUrl);
+   * 
+   * // Wait for the job to complete.
+   * try {
+   *   Job completedJob = job.waitFor(WaitForOption.checkEvery(1, TimeUnit.SECONDS),
+   *       WaitForOption.timeout(60, TimeUnit.SECONDS));
+   *   if (completedJob != null && completedJob.status().error() == null) {
+   *     // Job completed successfully.
+   *   } else {
+   *     // Handle error case.
+   *   }
+   * } catch(InterruptedException | TimeoutException e) {
+   *   // Handle interrupted wait.
+   * }
+   * }
+ * * @param format the format of the extracted data * @param destinationUri the fully-qualified Google Cloud Storage URI (e.g. gs://bucket/path) * where the extracted table should be written @@ -268,6 +398,31 @@ public Job extract(String format, String destinationUri, JobOption... options) * Starts a BigQuery Job to extract the current table to the provided destination URIs. Returns * the started {@link Job} object. * + *

Example extracting data to a list of Google Cloud Storage files. + * 

 {@code
+   * String format = "CSV";
+   * String gcsUrl1 = "gs://bucket_name/PartitionA_*.csv";
+   * String gcsUrl2 = "gs://bucket_name/PartitionB_*.csv";
+   * List destinationUris = new ArrayList<>();
+   * destinationUris.add(gcsUrl1);
+   * destinationUris.add(gcsUrl2);
+   * 
+   * Job job = table.extract(format, destinationUris);
+   * 
+   * // Wait for the job to complete.
+   * try {
+   *   Job completedJob = job.waitFor(WaitForOption.checkEvery(1, TimeUnit.SECONDS),
+   *       WaitForOption.timeout(60, TimeUnit.SECONDS));
+   *   if (completedJob != null && completedJob.status().error() == null) {
+   *     // Job completed successfully.
+   *   } else {
+   *     // Handle error case.
+   *   }
+   * } catch(InterruptedException | TimeoutException e) {
+   *   // Handle interrupted wait.
+   * }
+   * }
+ * * @param format the format of the exported data * @param destinationUris the fully-qualified Google Cloud Storage URIs (e.g. gs://bucket/path) * where the extracted table should be written diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java index 4d3162c690e1..3aa637671305 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java @@ -67,12 +67,12 @@ public void checkExists() { /** * Example of fetching a table's latest information, specifying particular table field options. */ - // [TARGET reload(BigQuery.TableOption... options)] + // [TARGET reload(TableOption...)] // [VARIABLE TableField.LAST_MODIFIED_TIME] // [VARIABLE TableField.NUM_ROWS] - public Table reloadTableWithFields(TableField... fields) { + public Table reloadTableWithFields(TableField field1, TableField field2) { // [START reloadTableWithFields] - Table reloaded = table.reload(TableOption.fields(fields)); + Table reloaded = table.reload(TableOption.fields(field1, field2)); // [END reloadTableWithFields] return reloaded; } @@ -80,12 +80,12 @@ public Table reloadTableWithFields(TableField... fields) { /** * Example of updating a table's information, specifying particular table field options. */ - // [TARGET update(BigQuery.TableOption... options)] + // [TARGET update(TableOption...)] // [VARIABLE TableField.LAST_MODIFIED_TIME] // [VARIABLE TableField.NUM_ROWS] - public Table updateTableWithFields(TableField... fields) { + public Table updateTableWithFields(TableField field1, TableField field2) { // [START updateTableWithFields] - Table updated = table.update(TableOption.fields(fields)); + Table updated = table.update(TableOption.fields(field1, field2)); // [END updateTableWithFields] return updated; } From ba889115d4072bc0c6b9d64da717afb1ad66efde Mon Sep 17 00:00:00 2001 From: Daniel Tang Date: Fri, 16 Sep 2016 17:38:59 -0700 Subject: [PATCH 25/28] run formatter over code --- .../bigquery/snippets/TableSnippets.java | 50 +++++++++---------- .../bigquery/snippets/ITTableSnippets.java | 46 +++++++++-------- 2 files changed, 49 insertions(+), 47 deletions(-) diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java index 3aa637671305..26d4609d23fe 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java @@ -53,15 +53,15 @@ public TableSnippets(Table table) { } /** - * Example of ensuring that a table exists. + * Example of ensuring that a table exists. */ // [TARGET exists()] public void checkExists() { - // [START checkExists] - if (!table.exists()) { - throw new RuntimeException("Table does not exist."); - } - // [END checkExists] + // [START checkExists] + if (!table.exists()) { + throw new RuntimeException("Table does not exist."); + } + // [END checkExists] } /** @@ -71,10 +71,10 @@ public void checkExists() { // [VARIABLE TableField.LAST_MODIFIED_TIME] // [VARIABLE TableField.NUM_ROWS] public Table reloadTableWithFields(TableField field1, TableField field2) { - // [START reloadTableWithFields] - Table reloaded = table.reload(TableOption.fields(field1, field2)); - // [END reloadTableWithFields] - return reloaded; + // [START reloadTableWithFields] + Table reloaded = table.reload(TableOption.fields(field1, field2)); + // [END reloadTableWithFields] + return reloaded; } /** @@ -84,10 +84,10 @@ public Table reloadTableWithFields(TableField field1, TableField field2) { // [VARIABLE TableField.LAST_MODIFIED_TIME] // [VARIABLE TableField.NUM_ROWS] public Table updateTableWithFields(TableField field1, TableField field2) { - // [START updateTableWithFields] - Table updated = table.update(TableOption.fields(field1, field2)); - // [END updateTableWithFields] - return updated; + // [START updateTableWithFields] + Table updated = table.update(TableOption.fields(field1, field2)); + // [END updateTableWithFields] + return updated; } /** @@ -95,11 +95,11 @@ public Table updateTableWithFields(TableField field1, TableField field2) { */ // [TARGET delete()] public void delete() { - // [START delete] - table.delete(); - // [END delete] + // [START delete] + table.delete(); + // [END delete] } - + /** * Example of inserting rows into a table. */ @@ -207,13 +207,13 @@ public Job copyTableId(String dataset, String tableName) throws BigQueryExceptio } else { // Handle error case. } - } catch(InterruptedException | TimeoutException e) { + } catch (InterruptedException | TimeoutException e) { // Handle interrupted wait. } // [END copyTableId] return job; } - + /** * Example extracting data to a list of Google Cloud Storage files. */ @@ -238,7 +238,7 @@ public Job extractList(String format, String gcsUrl1, String gcsUrl2) { } else { // Handle error case. } - } catch(InterruptedException | TimeoutException e) { + } catch (InterruptedException | TimeoutException e) { // Handle interrupted wait. } // [END extractList] @@ -264,7 +264,7 @@ public Job extractSingle(String format, String gcsUrl) { } else { // Handle error case. } - } catch(InterruptedException | TimeoutException e) { + } catch (InterruptedException | TimeoutException e) { // Handle interrupted wait. } // [END extractSingle] @@ -294,7 +294,7 @@ public Job loadList(String gcsUrl1, String gcsUrl2) { } else { // Handle error case. } - } catch(InterruptedException | TimeoutException e) { + } catch (InterruptedException | TimeoutException e) { // Handle interrupted wait. } // [END loadList] @@ -319,10 +319,10 @@ public Job loadSingle(String sourceUri) { } else { // Handle error case. } - } catch(InterruptedException | TimeoutException e) { + } catch (InterruptedException | TimeoutException e) { // Handle interrupted wait. } // [END loadSingle] return job; } -} \ No newline at end of file +} diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java index 6e84ad8e00b1..32dc59d9bea9 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java @@ -74,12 +74,12 @@ public class ITTableSnippets { private static final String DOOMED_TABLE_NAME = "doomed_table"; private static final String DOOMED_DATASET_NAME = "doomed_dataset"; public static final TableId DOOMED_TABLE_ID = TableId.of(DOOMED_DATASET_NAME, DOOMED_TABLE_NAME); - + private static Table doomedTable; private static TableSnippets doomedTableSnippets; private static int nextTableNumber = 0; - + @BeforeClass public static void beforeClass() { bigquery = BigQueryOptions.defaultInstance().service(); @@ -97,7 +97,7 @@ public void before() { table = bigquery.create(TableInfo.of(getTableId(), builder.build())); bigquery.create(TableInfo.of(getCopyTableId(), builder.build())); tableSnippets = new TableSnippets(table); - + doomedTable = bigquery.create(TableInfo.of(DOOMED_TABLE_ID, builder.build())); doomedTableSnippets = new TableSnippets(doomedTable); } @@ -119,7 +119,7 @@ public static void afterClass() { private String getTableName() { return BASE_TABLE_NAME + nextTableNumber; } - + private TableId getTableId() { return TableId.of(DATASET_NAME, getTableName()); } @@ -134,31 +134,31 @@ private TableId getCopyTableId() { @Test public void testCheckExists() { - log.info("testCheckExists"); - tableSnippets.checkExists(); + log.info("testCheckExists"); + tableSnippets.checkExists(); } - + @Test public void testReloadTableWithFields() { - log.info("testReloadTableWithFields"); - tableSnippets.reloadTableWithFields(TableField.LAST_MODIFIED_TIME, TableField.NUM_ROWS); + log.info("testReloadTableWithFields"); + tableSnippets.reloadTableWithFields(TableField.LAST_MODIFIED_TIME, TableField.NUM_ROWS); } - + @Test public void testUpdateTableWithFields() { - log.info("testUpdateTableWithFields"); - tableSnippets.updateTableWithFields(TableField.LAST_MODIFIED_TIME, TableField.NUM_ROWS); + log.info("testUpdateTableWithFields"); + tableSnippets.updateTableWithFields(TableField.LAST_MODIFIED_TIME, TableField.NUM_ROWS); } - + @Test public void testDelete() { - log.info("testDelete"); - doomedTableSnippets.delete(); + log.info("testDelete"); + doomedTableSnippets.delete(); } @Test public void testInsert() { - log.info("testInsert"); + log.info("testInsert"); InsertAllResponse response = tableSnippets.insert("row1", "row2"); assertFalse(response.hasErrors()); verifyTestRows(table); @@ -231,7 +231,7 @@ public String toString() { return ""; } } - + /** * Inserts some data into the test table. */ @@ -253,9 +253,10 @@ private void insertTestRows() { response = table.insert(rows); } } - + /** * Verifies that the given table has the rows inserted by InsertTestRows(). + * * @param checkTable The table to query. */ private void verifyTestRows(Table checkTable) { @@ -272,8 +273,9 @@ public Value apply(List row) { } /** - * Waits for a specified number of rows to appear in the given table. This is used - * by verifyTestRows to wait for data to appear before verifying. + * Waits for a specified number of rows to appear in the given table. This is used by + * verifyTestRows to wait for data to appear before verifying. + * * @param checkTable * @param numRows * @return The rows from the table. @@ -290,14 +292,14 @@ private List> waitForTableRows(Table checkTable, int numRows) { try { ++numSleeps; Thread.sleep(5000); - } catch(InterruptedException e) { + } catch (InterruptedException e) { } page = checkTable.list(TableDataListOption.pageSize(100)); rows = ImmutableList.copyOf(page.values()); } return rows; } - + @Test public void testCopyTableId() { log.info("testCopyTableId"); From 9733dcc96ae9879975b8b91c82708321dd854843 Mon Sep 17 00:00:00 2001 From: Daniel Tang Date: Fri, 16 Sep 2016 17:41:06 -0700 Subject: [PATCH 26/28] update Table javadoc --- .../java/com/google/cloud/bigquery/Table.java | 57 ++++++++++++++++--- .../bigquery/snippets/TableSnippets.java | 4 +- 2 files changed, 52 insertions(+), 9 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Table.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Table.java index 0568a1ae8e16..0a980eb21df4 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Table.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Table.java @@ -142,7 +142,7 @@ public Table build() { *

Example of ensuring that a table exists. * 

 {@code
    * if (!table.exists()) {
-   *  throw new RuntimeException("Table does not exist.");
+   *   throw new RuntimeException("Table does not exist.");
    * }
    * }
* @@ -344,7 +344,7 @@ public Job copy(String destinationDataset, String destinationTable, JobOption... * } else { * // Handle error case. * } - * } catch(InterruptedException | TimeoutException e) { + * } catch (InterruptedException | TimeoutException e) { * // Handle interrupted wait. * } * } @@ -366,7 +366,7 @@ public Job copy(TableId destinationTable, JobOption... options) *

Example extracting data to single Google Cloud Storage file. * 

 {@code
    * String format = "CSV";
-   * String gcsUrl = "gs://bucket_name/filename.csv";
+   * String gcsUrl = "gs://myapp.appspot.com/filename.csv";
    * Job job = table.extract(format, gcsUrl);
    * 
    * // Wait for the job to complete.
@@ -378,7 +378,7 @@ public Job copy(TableId destinationTable, JobOption... options)
    *   } else {
    *     // Handle error case.
    *   }
-   * } catch(InterruptedException | TimeoutException e) {
+   * } catch (InterruptedException | TimeoutException e) {
    *   // Handle interrupted wait.
    * }
    * }
@@ -401,8 +401,8 @@ public Job extract(String format, String destinationUri, JobOption... options) *

Example extracting data to a list of Google Cloud Storage files. * 

 {@code
    * String format = "CSV";
-   * String gcsUrl1 = "gs://bucket_name/PartitionA_*.csv";
-   * String gcsUrl2 = "gs://bucket_name/PartitionB_*.csv";
+   * String gcsUrl1 = "gs://myapp.appspot.com/PartitionA_*.csv";
+   * String gcsUrl2 = "gs://myapp.appspot.com/PartitionB_*.csv";
    * List destinationUris = new ArrayList<>();
    * destinationUris.add(gcsUrl1);
    * destinationUris.add(gcsUrl2);
@@ -418,7 +418,7 @@ public Job extract(String format, String destinationUri, JobOption... options)
    *   } else {
    *     // Handle error case.
    *   }
-   * } catch(InterruptedException | TimeoutException e) {
+   * } catch (InterruptedException | TimeoutException e) {
    *   // Handle interrupted wait.
    * }
    * }
@@ -440,6 +440,25 @@ public Job extract(String format, List destinationUris, JobOption... opt * Starts a BigQuery Job to load data into the current table from the provided source URI. Returns * the started {@link Job} object. * + *

Example loading data from a single Google Cloud Storage file. + * 

 {@code
+   * String sourceUri = "gs://myapp.appspot.com/filename.csv";
+   * Job job = table.load(FormatOptions.csv(), sourceUri);
+   * 
+   * // Wait for the job to complete.
+   * try {
+   *   Job completedJob = job.waitFor(WaitForOption.checkEvery(1, TimeUnit.SECONDS),
+   *       WaitForOption.timeout(60, TimeUnit.SECONDS));
+   *   if (completedJob != null && completedJob.status().error() == null) {
+   *     // Job completed successfully.
+   *   } else {
+   *     // Handle error case.
+   *   }
+   * } catch (InterruptedException | TimeoutException e) {
+   *   // Handle interrupted wait.
+   * }
+   * }
+ * * @param format the format of the data to load * @param sourceUri the fully-qualified Google Cloud Storage URI (e.g. gs://bucket/path) from * which to load the data @@ -455,6 +474,30 @@ public Job load(FormatOptions format, String sourceUri, JobOption... options) * Starts a BigQuery Job to load data into the current table from the provided source URIs. * Returns the started {@link Job} object. * + *

Example loading data from a list of Google Cloud Storage files. + * 

 {@code
+   * String gcsUrl1 = "gs://myapp.appspot.com/PartitionA_000000000000.csv";
+   * String gcsUrl2 = "gs://myapp.appspot.com/PartitionB_000000000000.csv";
+   * List sourceUris = new ArrayList<>();
+   * sourceUris.add(gcsUrl1);
+   * sourceUris.add(gcsUrl2);
+   * 
+   * Job job = table.load(FormatOptions.csv(), sourceUris);
+   * 
+   * // Wait for the job to complete.
+   * try {
+   *   Job completedJob = job.waitFor(WaitForOption.checkEvery(1, TimeUnit.SECONDS),
+   *       WaitForOption.timeout(60, TimeUnit.SECONDS));
+   *   if (completedJob != null && completedJob.status().error() == null) {
+   *     // Job completed successfully.
+   *   } else {
+   *     // Handle error case.
+   *   }
+   * } catch (InterruptedException | TimeoutException e) {
+   *   // Handle interrupted wait.
+   * }
+   * }
+ * * @param format the format of the exported data * @param sourceUris the fully-qualified Google Cloud Storage URIs (e.g. gs://bucket/path) from * which to load the data diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java index 26d4609d23fe..3fc851b68f6a 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java @@ -274,7 +274,7 @@ public Job extractSingle(String format, String gcsUrl) { /** * Example loading data from a list of Google Cloud Storage files. */ - // [TARGET load(String, List, JobOption...)] + // [TARGET load(FormatOptions, List, JobOption...)] // [VARIABLE "gs://myapp.appspot.com/PartitionA_000000000000.csv"] // [VARIABLE "gs://myapp.appspot.com/PartitionB_000000000000.csv"] public Job loadList(String gcsUrl1, String gcsUrl2) { @@ -304,7 +304,7 @@ public Job loadList(String gcsUrl1, String gcsUrl2) { /** * Example loading data from a single Google Cloud Storage file. */ - // [TARGET load(String, String, JobOption...)] + // [TARGET load(FormatOptions, String, JobOption...)] // [VARIABLE "gs://myapp.appspot.com/filename.csv"] public Job loadSingle(String sourceUri) { // [START loadSingle] From 84904d82779a7e3cf7f1a7438b126a1e0b961f4c Mon Sep 17 00:00:00 2001 From: Daniel Tang Date: Mon, 19 Sep 2016 10:57:45 -0700 Subject: [PATCH 27/28] TableSnippets: cleanup unused imports --- .../bigquery/snippets/TableSnippets.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java index 3fc851b68f6a..3314c675dce9 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java @@ -16,14 +16,20 @@ package com.google.cloud.examples.bigquery.snippets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + import com.google.cloud.Page; import com.google.cloud.WaitForOption; import com.google.cloud.bigquery.BigQuery.JobField; import com.google.cloud.bigquery.BigQuery.JobOption; -import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.BigQuery.TableOption; -import com.google.cloud.bigquery.BigQuery.TableField; import com.google.cloud.bigquery.BigQuery.TableDataListOption; +import com.google.cloud.bigquery.BigQuery.TableField; +import com.google.cloud.bigquery.BigQuery.TableOption; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.FieldValue; import com.google.cloud.bigquery.FormatOptions; @@ -32,13 +38,6 @@ import com.google.cloud.bigquery.Job; import com.google.cloud.bigquery.Table; import com.google.cloud.bigquery.TableId; -import com.google.cloud.bigquery.spi.BigQueryRpc; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; /* * EDITING INSTRUCTIONS From 2647a4e3792557f7e2d4a5f5562e9a675edbe9f1 Mon Sep 17 00:00:00 2001 From: Daniel Tang Date: Mon, 19 Sep 2016 11:20:05 -0700 Subject: [PATCH 28/28] cleanup ITTableSnippets --- .../bigquery/snippets/TableSnippets.java | 2 +- .../bigquery/snippets/ITTableSnippets.java | 130 +++++++----------- 2 files changed, 53 insertions(+), 79 deletions(-) diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java index 3314c675dce9..8f9e0d46bc91 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/TableSnippets.java @@ -58,7 +58,7 @@ public TableSnippets(Table table) { public void checkExists() { // [START checkExists] if (!table.exists()) { - throw new RuntimeException("Table does not exist."); + throw new IllegalArgumentException("Table does not exist."); } // [END checkExists] } diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java index 32dc59d9bea9..e2b313112820 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITTableSnippets.java @@ -20,10 +20,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.logging.Logger; @@ -43,7 +40,6 @@ import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.Field.Type; import com.google.cloud.bigquery.FieldValue; -import com.google.cloud.bigquery.InsertAllRequest.RowToInsert; import com.google.cloud.bigquery.InsertAllResponse; import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardTableDefinition; @@ -78,7 +74,7 @@ public class ITTableSnippets { private static Table doomedTable; private static TableSnippets doomedTableSnippets; - private static int nextTableNumber = 0; + private static int nextTableNumber; @BeforeClass public static void beforeClass() { @@ -169,7 +165,7 @@ public void testInsertParams() throws InterruptedException { InsertAllResponse response = tableSnippets.insertWithParams("row1", "row2"); assertTrue(response.hasErrors()); List> rows = ImmutableList.copyOf(tableSnippets.list().values()); - while (rows.size() == 0) { + while (rows.isEmpty()) { Thread.sleep(500); rows = ImmutableList.copyOf(tableSnippets.list().values()); } @@ -191,7 +187,7 @@ public void testList() throws InterruptedException { InsertAllResponse response = tableSnippets.insert("row1", "row2"); assertFalse(response.hasErrors()); rows = ImmutableList.copyOf(tableSnippets.list().values()); - while (rows.size() == 0) { + while (rows.isEmpty()) { Thread.sleep(500); rows = ImmutableList.copyOf(tableSnippets.list().values()); } @@ -203,6 +199,55 @@ public void testCopy() { tableSnippets.copy(COPY_DATASET_NAME, BASE_TABLE_NAME); } + @Test + public void testCopyTableId() { + log.info("testCopyTableId"); + tableSnippets.copyTableId(COPY_DATASET_NAME, getCopyTableName()); + } + + @Test + public void testExtractList() { + log.info("testExtractList"); + String projectId = bigquery.options().projectId(); + String gcsFile1 = "gs://" + projectId + ".appspot.com/extractTestA_*.csv"; + String gcsFile2 = "gs://" + projectId + ".appspot.com/extractTestB_*.csv"; + tableSnippets.extractList("CSV", gcsFile1, gcsFile2); + } + + @Test + public void testExtractSingle() { + log.info("testExtractSingle"); + String projectId = bigquery.options().projectId(); + String gcsFile = "gs://" + projectId + ".appspot.com/extractTest.csv"; + tableSnippets.extractSingle("CSV", gcsFile); + } + + @Test + public void testLoadList() { + log.info("testLoadList"); + String projectId = bigquery.options().projectId(); + String gcsFile1 = "gs://" + projectId + ".appspot.com/loadTest1.csv"; + String gcsFile2 = "gs://" + projectId + ".appspot.com/loadTest2.csv"; + + // Before we can load, we should make sure those files exist. + tableSnippets.extractSingle("CSV", gcsFile1); + tableSnippets.extractSingle("CSV", gcsFile2); + + tableSnippets.loadList(gcsFile1, gcsFile2); + } + + @Test + public void testLoadSingle() { + log.info("testLoadSingle"); + String projectId = bigquery.options().projectId(); + String gcsFile = "gs://" + projectId + ".appspot.com/loadSingle.csv"; + + // Before we can load, we should make sure the file exists. + tableSnippets.extractSingle("CSV", gcsFile); + + tableSnippets.loadSingle(gcsFile); + } + private static class Value { final String stringField; final boolean booleanField; @@ -232,28 +277,6 @@ public String toString() { } } - /** - * Inserts some data into the test table. - */ - private void insertTestRows() { - String rowId1 = "row1"; - String rowId2 = "row2"; - List rows = new ArrayList<>(); - Map row1 = new HashMap<>(); - row1.put("stringField", ROW1.stringField); - row1.put("booleanField", ROW1.booleanField); - Map row2 = new HashMap<>(); - row2.put("stringField", ROW2.stringField); - row2.put("booleanField", ROW2.booleanField); - rows.add(RowToInsert.of(rowId1, row1)); - rows.add(RowToInsert.of(rowId2, row2)); - InsertAllResponse response = table.insert(rows); - while (response.hasErrors()) { - log.info("Error inserting rows. Trying again..."); - response = table.insert(rows); - } - } - /** * Verifies that the given table has the rows inserted by InsertTestRows(). * @@ -299,53 +322,4 @@ private List> waitForTableRows(Table checkTable, int numRows) { } return rows; } - - @Test - public void testCopyTableId() { - log.info("testCopyTableId"); - tableSnippets.copyTableId(COPY_DATASET_NAME, getCopyTableName()); - } - - @Test - public void testExtractList() { - log.info("testExtractList"); - String projectId = bigquery.options().projectId(); - String gcsFile1 = "gs://" + projectId + ".appspot.com/extractTestA_*.csv"; - String gcsFile2 = "gs://" + projectId + ".appspot.com/extractTestB_*.csv"; - tableSnippets.extractList("CSV", gcsFile1, gcsFile2); - } - - @Test - public void testExtractSingle() { - log.info("testExtractSingle"); - String projectId = bigquery.options().projectId(); - String gcsFile = "gs://" + projectId + ".appspot.com/extractTest.csv"; - tableSnippets.extractSingle("CSV", gcsFile); - } - - @Test - public void testLoadList() { - log.info("testLoadList"); - String projectId = bigquery.options().projectId(); - String gcsFile1 = "gs://" + projectId + ".appspot.com/loadTest1.csv"; - String gcsFile2 = "gs://" + projectId + ".appspot.com/loadTest2.csv"; - - // Before we can load, we should make sure those files exist. - tableSnippets.extractSingle("CSV", gcsFile1); - tableSnippets.extractSingle("CSV", gcsFile2); - - tableSnippets.loadList(gcsFile1, gcsFile2); - } - - @Test - public void testLoadSingle() { - log.info("testLoadSingle"); - String projectId = bigquery.options().projectId(); - String gcsFile = "gs://" + projectId + ".appspot.com/loadSingle.csv"; - - // Before we can load, we should make sure the file exists. - tableSnippets.extractSingle("CSV", gcsFile); - - tableSnippets.loadSingle(gcsFile); - } }