From acbd5c0bfb57e3688a376a695e6a5893396c852e Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Tue, 29 Dec 2015 17:16:58 +0100 Subject: [PATCH] Add support for templateSuffix to BigQuery InsertAllRequest --- .../google/gcloud/bigquery/BigQueryImpl.java | 1 + .../gcloud/bigquery/InsertAllRequest.java | 36 ++++++++++++- .../gcloud/bigquery/BigQueryImplTest.java | 3 +- .../gcloud/bigquery/ITBigQueryTest.java | 35 ++++++++++++ .../gcloud/bigquery/InsertAllRequestTest.java | 53 ++++++++++++++----- 5 files changed, 111 insertions(+), 17 deletions(-) diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQueryImpl.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQueryImpl.java index 62685d8ecc46..a65490998c17 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQueryImpl.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQueryImpl.java @@ -412,6 +412,7 @@ public InsertAllResponse insertAll(InsertAllRequest request) throws BigQueryExce final TableDataInsertAllRequest requestPb = new TableDataInsertAllRequest(); requestPb.setIgnoreUnknownValues(request.ignoreUnknownValues()); requestPb.setSkipInvalidRows(request.skipInvalidRows()); + requestPb.setTemplateSuffix(request.templateSuffix()); List rowsPb = Lists.transform(request.rows(), new Function() { @Override public Rows apply(RowToInsert rowToInsert) { diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/InsertAllRequest.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/InsertAllRequest.java index 56be098b197b..8b6f573b1e3c 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/InsertAllRequest.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/InsertAllRequest.java @@ -45,6 +45,7 @@ public class InsertAllRequest implements Serializable { private final List rows; private final Boolean skipInvalidRows; private final Boolean ignoreUnknownValues; + private final String templateSuffix; /** * A Google Big Query row to be inserted into a table. Each {@code RowToInsert} has an associated @@ -140,6 +141,7 @@ public static final class Builder { private List rows; private Boolean skipInvalidRows; private Boolean ignoreUnknownValues; + private String templateSuffix; private Builder() {} @@ -231,6 +233,20 @@ public Builder ignoreUnknownValues(boolean ignoreUnknownValues) { return this; } + /** + * If specified, the destination table is treated as a base template. Rows are inserted into an + * instance table named "{destination}{templateSuffix}". BigQuery will manage the creation of + * the instance table, using the schema of the base template table. + * + * @see + * Template Tables + */ + public Builder templateSuffix(String templateSuffix) { + this.templateSuffix = templateSuffix; + return this; + } + public InsertAllRequest build() { return new InsertAllRequest(this); } @@ -241,6 +257,7 @@ private InsertAllRequest(Builder builder) { this.rows = ImmutableList.copyOf(checkNotNull(builder.rows)); this.ignoreUnknownValues = builder.ignoreUnknownValues; this.skipInvalidRows = builder.skipInvalidRows; + this.templateSuffix = builder.templateSuffix; } /** @@ -273,6 +290,19 @@ public Boolean skipInvalidRows() { return skipInvalidRows; } + /** + * If specified, the destination table is treated as a base template. Rows are inserted into an + * instance table named "{destination}{templateSuffix}". BigQuery will manage the creation of the + * instance table, using the schema of the base template table. + * + * @see + * Template Tables + */ + public String templateSuffix() { + return templateSuffix; + } + /** * Returns a builder for an {@code InsertAllRequest} object given the destination table. */ @@ -384,12 +414,13 @@ public String toString() { .add("rows", rows) .add("ignoreUnknownValues", ignoreUnknownValues) .add("skipInvalidRows", skipInvalidRows) + .add("templateSuffix", templateSuffix) .toString(); } @Override public int hashCode() { - return Objects.hash(table, rows, ignoreUnknownValues, skipInvalidRows); + return Objects.hash(table, rows, ignoreUnknownValues, skipInvalidRows, templateSuffix); } @Override @@ -401,6 +432,7 @@ public boolean equals(Object obj) { return Objects.equals(table, other.table) && Objects.equals(rows, other.rows) && Objects.equals(ignoreUnknownValues, other.ignoreUnknownValues) - && Objects.equals(skipInvalidRows, other.skipInvalidRows); + && Objects.equals(skipInvalidRows, other.skipInvalidRows) + && Objects.equals(templateSuffix, other.templateSuffix); } } diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/BigQueryImplTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/BigQueryImplTest.java index bcc946f65006..6a6a1c7cd6af 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/BigQueryImplTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/BigQueryImplTest.java @@ -614,6 +614,7 @@ public void testInsertAll() { .rows(rows) .skipInvalidRows(false) .ignoreUnknownValues(true) + .templateSuffix("suffix") .build(); TableDataInsertAllRequest requestPb = new TableDataInsertAllRequest().setRows( Lists.transform(rows, new Function() { @@ -623,7 +624,7 @@ public TableDataInsertAllRequest.Rows apply(RowToInsert rowToInsert) { .setJson(rowToInsert.content()); } }) - ).setSkipInvalidRows(false).setIgnoreUnknownValues(true); + ).setSkipInvalidRows(false).setIgnoreUnknownValues(true).setTemplateSuffix("suffix"); TableDataInsertAllResponse responsePb = new TableDataInsertAllResponse().setInsertErrors( ImmutableList.of(new TableDataInsertAllResponse.InsertErrors().setIndex(0L).setErrors( ImmutableList.of(new ErrorProto().setMessage("ErrorMessage"))))); diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ITBigQueryTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ITBigQueryTest.java index 4a4f01de4124..34f4f6893187 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ITBigQueryTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ITBigQueryTest.java @@ -475,6 +475,41 @@ public void testInsertAll() { assertTrue(bigquery.delete(TableId.of(DATASET, tableName))); } + @Test + public void testInsertAllWithSuffix() { + String tableName = "test_insert_all_with_suffix_table"; + BaseTableInfo tableInfo = TableInfo.of(TableId.of(DATASET, tableName), TABLE_SCHEMA); + assertNotNull(bigquery.create(tableInfo)); + InsertAllRequest request = InsertAllRequest.builder(tableInfo.tableId()) + .addRow(ImmutableMap.of( + "TimestampField", "2014-08-19 07:41:35.220 -05:00", + "StringField", "stringValue", + "IntegerField", ImmutableList.of(0, 1), + "BooleanField", false, + "RecordField", ImmutableMap.of( + "TimestampField", "1969-07-20 20:18:04 UTC", + "IntegerField", ImmutableList.of(1, 0), + "BooleanField", true))) + .addRow(ImmutableMap.of( + "TimestampField", "2014-08-19 07:41:35.220 -05:00", + "StringField", "stringValue", + "IntegerField", ImmutableList.of(0, 1), + "BooleanField", false, + "RecordField", ImmutableMap.of( + "TimestampField", "1969-07-20 20:18:04 UTC", + "IntegerField", ImmutableList.of(1, 0), + "BooleanField", true))) + .templateSuffix("_suffix") + .build(); + InsertAllResponse response = bigquery.insertAll(request); + assertFalse(response.hasErrors()); + assertEquals(0, response.insertErrors().size()); + String newTableName = tableName + "_suffix"; + assertNotNull(bigquery.getTable(DATASET, newTableName, TableOption.fields())); + assertTrue(bigquery.delete(TableId.of(DATASET, tableName))); + assertTrue(bigquery.delete(TableId.of(DATASET, newTableName))); + } + @Test public void testInsertAllWithErrors() { String tableName = "test_insert_all_with_errors_table"; diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/InsertAllRequestTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/InsertAllRequestTest.java index fb744bd78920..d2e1de14a571 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/InsertAllRequestTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/InsertAllRequestTest.java @@ -18,6 +18,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import com.google.common.collect.ImmutableList; @@ -45,6 +46,7 @@ public class InsertAllRequestTest { private static final BaseTableInfo TABLE_INFO = TableInfo.of(TABLE_ID, TABLE_SCHEMA); private static final boolean SKIP_INVALID_ROWS = true; private static final boolean IGNORE_UNKNOWN_VALUES = false; + private static final String TEMPLATE_SUFFIX = "templateSuffix"; private static final InsertAllRequest INSERT_ALL_REQUEST1 = InsertAllRequest.builder(TABLE_ID) .addRow(CONTENT1) .addRow(CONTENT2) @@ -90,20 +92,25 @@ public class InsertAllRequestTest { .ignoreUnknownValues(IGNORE_UNKNOWN_VALUES) .skipInvalidRows(SKIP_INVALID_ROWS) .build(); - private static final InsertAllRequest INSERT_ALL_REQUEST9 = - InsertAllRequest.builder(TABLE_INFO) - .addRow("id1", CONTENT1) - .addRow("id2", CONTENT2) - .ignoreUnknownValues(IGNORE_UNKNOWN_VALUES) - .skipInvalidRows(SKIP_INVALID_ROWS) - .build(); - private static final InsertAllRequest INSERT_ALL_REQUEST10 = - InsertAllRequest.builder(TABLE_INFO) - .addRow("id1", CONTENT1) - .addRow("id2", CONTENT2) - .ignoreUnknownValues(true) - .skipInvalidRows(false) - .build(); + private static final InsertAllRequest INSERT_ALL_REQUEST9 = InsertAllRequest.builder(TABLE_INFO) + .addRow("id1", CONTENT1) + .addRow("id2", CONTENT2) + .ignoreUnknownValues(IGNORE_UNKNOWN_VALUES) + .skipInvalidRows(SKIP_INVALID_ROWS) + .build(); + private static final InsertAllRequest INSERT_ALL_REQUEST10 = InsertAllRequest.builder(TABLE_INFO) + .addRow("id1", CONTENT1) + .addRow("id2", CONTENT2) + .ignoreUnknownValues(true) + .skipInvalidRows(false) + .build(); + private static final InsertAllRequest INSERT_ALL_REQUEST11 = InsertAllRequest.builder(TABLE_INFO) + .addRow("id1", CONTENT1) + .addRow("id2", CONTENT2) + .ignoreUnknownValues(true) + .skipInvalidRows(false) + .templateSuffix(TEMPLATE_SUFFIX) + .build(); @Test public void testBuilder() { @@ -117,6 +124,7 @@ public void testBuilder() { assertEquals(TABLE_ID, INSERT_ALL_REQUEST8.table()); assertEquals(TABLE_ID, INSERT_ALL_REQUEST9.table()); assertEquals(TABLE_ID, INSERT_ALL_REQUEST10.table()); + assertEquals(TABLE_ID, INSERT_ALL_REQUEST11.table()); assertEquals(ROWS, INSERT_ALL_REQUEST1.rows()); assertEquals(ROWS, INSERT_ALL_REQUEST2.rows()); assertEquals(ROWS, INSERT_ALL_REQUEST4.rows()); @@ -127,6 +135,7 @@ public void testBuilder() { assertEquals(ROWS_WITH_ID, INSERT_ALL_REQUEST8.rows()); assertEquals(ROWS_WITH_ID, INSERT_ALL_REQUEST9.rows()); assertEquals(ROWS_WITH_ID, INSERT_ALL_REQUEST10.rows()); + assertEquals(ROWS_WITH_ID, INSERT_ALL_REQUEST11.rows()); assertEquals(SKIP_INVALID_ROWS, INSERT_ALL_REQUEST1.skipInvalidRows()); assertEquals(SKIP_INVALID_ROWS, INSERT_ALL_REQUEST2.skipInvalidRows()); assertEquals(SKIP_INVALID_ROWS, INSERT_ALL_REQUEST3.skipInvalidRows()); @@ -137,6 +146,7 @@ public void testBuilder() { assertEquals(SKIP_INVALID_ROWS, INSERT_ALL_REQUEST8.skipInvalidRows()); assertEquals(SKIP_INVALID_ROWS, INSERT_ALL_REQUEST9.skipInvalidRows()); assertFalse(INSERT_ALL_REQUEST10.skipInvalidRows()); + assertFalse(INSERT_ALL_REQUEST11.skipInvalidRows()); assertEquals(IGNORE_UNKNOWN_VALUES, INSERT_ALL_REQUEST1.ignoreUnknownValues()); assertEquals(IGNORE_UNKNOWN_VALUES, INSERT_ALL_REQUEST2.ignoreUnknownValues()); assertEquals(IGNORE_UNKNOWN_VALUES, INSERT_ALL_REQUEST3.ignoreUnknownValues()); @@ -147,6 +157,18 @@ public void testBuilder() { assertEquals(IGNORE_UNKNOWN_VALUES, INSERT_ALL_REQUEST8.ignoreUnknownValues()); assertEquals(IGNORE_UNKNOWN_VALUES, INSERT_ALL_REQUEST9.ignoreUnknownValues()); assertTrue(INSERT_ALL_REQUEST10.ignoreUnknownValues()); + assertTrue(INSERT_ALL_REQUEST11.ignoreUnknownValues()); + assertNull(INSERT_ALL_REQUEST1.templateSuffix()); + assertNull(INSERT_ALL_REQUEST2.templateSuffix()); + assertNull(INSERT_ALL_REQUEST3.templateSuffix()); + assertNull(INSERT_ALL_REQUEST4.templateSuffix()); + assertNull(INSERT_ALL_REQUEST5.templateSuffix()); + assertNull(INSERT_ALL_REQUEST6.templateSuffix()); + assertNull(INSERT_ALL_REQUEST7.templateSuffix()); + assertNull(INSERT_ALL_REQUEST8.templateSuffix()); + assertNull(INSERT_ALL_REQUEST9.templateSuffix()); + assertNull(INSERT_ALL_REQUEST10.templateSuffix()); + assertEquals(TEMPLATE_SUFFIX, INSERT_ALL_REQUEST11.templateSuffix()); } @Test @@ -183,6 +205,8 @@ public void testEquals() { compareInsertAllRequest(INSERT_ALL_REQUEST5, INSERT_ALL_REQUEST7); compareInsertAllRequest(INSERT_ALL_REQUEST7, INSERT_ALL_REQUEST8); compareInsertAllRequest(INSERT_ALL_REQUEST8, INSERT_ALL_REQUEST9); + compareInsertAllRequest(INSERT_ALL_REQUEST10, INSERT_ALL_REQUEST10); + compareInsertAllRequest(INSERT_ALL_REQUEST11, INSERT_ALL_REQUEST11); } private void compareInsertAllRequest(InsertAllRequest expected, InsertAllRequest value) { @@ -193,5 +217,6 @@ private void compareInsertAllRequest(InsertAllRequest expected, InsertAllRequest assertEquals(expected.rows(), value.rows()); assertEquals(expected.ignoreUnknownValues(), value.ignoreUnknownValues()); assertEquals(expected.skipInvalidRows(), value.skipInvalidRows()); + assertEquals(expected.templateSuffix(), value.templateSuffix()); } }