diff --git a/gcloud-java-bigquery/pom.xml b/gcloud-java-bigquery/pom.xml index 87b0efdd3cff..660dd93916df 100644 --- a/gcloud-java-bigquery/pom.xml +++ b/gcloud-java-bigquery/pom.xml @@ -31,7 +31,7 @@ com.google.apis google-api-services-bigquery - v2-rev270-1.21.0 + v2-rev303-1.22.0 compile diff --git a/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/CsvOptions.java b/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/CsvOptions.java index b621ed2cc6bc..4273740f91d2 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/CsvOptions.java +++ b/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/CsvOptions.java @@ -34,7 +34,7 @@ public final class CsvOptions extends FormatOptions { private final String encoding; private final String fieldDelimiter; private final String quote; - private final Integer skipLeadingRows; + private final Long skipLeadingRows; public static final class Builder { @@ -43,10 +43,19 @@ public static final class Builder { private String encoding; private String fieldDelimiter; private String quote; - private Integer skipLeadingRows; + private Long skipLeadingRows; private Builder() {} + private Builder(CsvOptions csvOptions) { + this.allowJaggedRows = csvOptions.allowJaggedRows; + this.allowQuotedNewLines = csvOptions.allowQuotedNewLines; + this.encoding = csvOptions.encoding; + this.fieldDelimiter = csvOptions.fieldDelimiter; + this.quote = csvOptions.quote; + this.skipLeadingRows = csvOptions.skipLeadingRows; + } + /** * Set whether BigQuery should accept rows that are missing trailing optional columns. If * {@code true}, BigQuery treats missing trailing columns as null values. If {@code false}, @@ -54,7 +63,7 @@ private Builder() {} * bad records, an invalid error is returned in the job result. By default, rows with missing * trailing columns are considered bad records. */ - public Builder allowJaggedRows(Boolean allowJaggedRows) { + public Builder allowJaggedRows(boolean allowJaggedRows) { this.allowJaggedRows = allowJaggedRows; return this; } @@ -63,7 +72,7 @@ public Builder allowJaggedRows(Boolean allowJaggedRows) { * Sets whether BigQuery should allow quoted data sections that contain newline characters in a * CSV file. By default quoted newline are not allowed. */ - public Builder allowQuotedNewLines(Boolean allowQuotedNewLines) { + public Builder allowQuotedNewLines(boolean allowQuotedNewLines) { this.allowQuotedNewLines = allowQuotedNewLines; return this; } @@ -104,7 +113,7 @@ public Builder fieldDelimiter(String fieldDelimiter) { * string to ISO-8859-1 encoding, and then uses the first byte of the encoded string to split * the data in its raw, binary state. The default value is a double-quote ('"'). If your data * does not contain quoted sections, set the property value to an empty string. If your data - * contains quoted newline characters, you must also set {@link #allowQuotedNewLines(Boolean)} + * contains quoted newline characters, you must also set {@link #allowQuotedNewLines(boolean)} * property to {@code true}. */ public Builder quote(String quote) { @@ -117,7 +126,7 @@ public Builder quote(String quote) { * data. The default value is 0. This property is useful if you have header rows in the file * that should be skipped. */ - public Builder skipLeadingRows(Integer skipLeadingRows) { + public Builder skipLeadingRows(long skipLeadingRows) { this.skipLeadingRows = skipLeadingRows; return this; } @@ -186,7 +195,7 @@ public String quote() { * Returns the number of rows at the top of a CSV file that BigQuery will skip when reading the * data. */ - public Integer skipLeadingRows() { + public Long skipLeadingRows() { return skipLeadingRows; } @@ -194,13 +203,7 @@ public Integer skipLeadingRows() { * Returns a builder for the {@code CsvOptions} object. */ public Builder toBuilder() { - return new Builder() - .allowJaggedRows(allowJaggedRows) - .allowQuotedNewLines(allowQuotedNewLines) - .encoding(encoding) - .fieldDelimiter(fieldDelimiter) - .quote(quote) - .skipLeadingRows(skipLeadingRows); + return new Builder(this); } @Override diff --git a/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/Field.java b/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/Field.java index dc805e12c2a2..874049aa033a 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/Field.java +++ b/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/Field.java @@ -73,7 +73,7 @@ public static class Type implements Serializable { private static final long serialVersionUID = 2841484762609576959L; public enum Value { - STRING, INTEGER, FLOAT, BOOLEAN, TIMESTAMP, RECORD + BYTES, STRING, INTEGER, FLOAT, BOOLEAN, TIMESTAMP, RECORD } private final Value value; @@ -108,6 +108,13 @@ public List fields() { return fields; } + /** + * Returns a {@link Value#BYTES} field value. + */ + public static Type bytes() { + return new Type(Value.BYTES); + } + /** * Returns a {@link Value#STRING} field value. */ diff --git a/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/FieldValue.java b/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/FieldValue.java index 1c06b87d639d..1eb3115a3644 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/FieldValue.java +++ b/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/FieldValue.java @@ -23,6 +23,7 @@ import com.google.common.base.Function; import com.google.common.base.MoreObjects; import com.google.common.collect.Lists; +import com.google.common.io.BaseEncoding; import java.io.Serializable; import java.util.List; @@ -54,7 +55,7 @@ public FieldValue apply(Object pb) { public enum Attribute { /** * A primitive field value. A {@code FieldValue} is primitive when the corresponding field has - * type {@link Field.Type#bool()}, {@link Field.Type#string()}, + * type {@link Field.Type#bytes()}, {@link Field.Type#bool()}, {@link Field.Type#string()}, * {@link Field.Type#floatingPoint()}, {@link Field.Type#integer()}, * {@link Field.Type#timestamp()} or the value is set to {@code null}. */ @@ -80,7 +81,7 @@ public enum Attribute { * Returns the attribute of this Field Value. * * @return {@link Attribute#PRIMITIVE} if the field is a primitive type - * ({@link Field.Type#bool()}, {@link Field.Type#string()}, + * ({@link Field.Type#bytes()}, {@link Field.Type#bool()}, {@link Field.Type#string()}, * {@link Field.Type#floatingPoint()}, {@link Field.Type#integer()}, * {@link Field.Type#timestamp()}) or is {@code null}. Returns {@link Attribute#REPEATED} if * the corresponding field has ({@link Field.Mode#REPEATED}) mode. Returns @@ -108,8 +109,8 @@ public Object value() { /** * Returns this field's value as a {@link String}. This method should only be used if the - * corresponding field has primitive type ({@link Field.Type#bool()}, {@link Field.Type#string()}, - * {@link Field.Type#floatingPoint()}, {@link Field.Type#integer()}, + * corresponding field has primitive type ({@link Field.Type#bytes()}, {@link Field.Type#bool()}, + * {@link Field.Type#string()}, {@link Field.Type#floatingPoint()}, {@link Field.Type#integer()}, * {@link Field.Type#timestamp()}). * * @throws ClassCastException if the field is not a primitive type @@ -121,6 +122,22 @@ public String stringValue() { return (String) value; } + /** + * Returns this field's value as a byte array. This method should only be used if the + * corresponding field has primitive type ({@link Field.Type#bytes()}. + * + * @throws ClassCastException if the field is not a primitive type + * @throws NullPointerException if {@link #isNull()} returns {@code true} + * @throws IllegalStateException if the field value is not encoded in base64 + */ + public byte[] bytesValue() { + try { + return BaseEncoding.base64().decode(stringValue()); + } catch (IllegalArgumentException ex) { + throw new IllegalStateException(ex); + } + } + /** * Returns this field's value as a {@code long}. This method should only be used if the * corresponding field has {@link Field.Type#integer()} type. diff --git a/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/InsertAllRequest.java b/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/InsertAllRequest.java index b46257833b78..569f6990d897 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/InsertAllRequest.java +++ b/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/InsertAllRequest.java @@ -49,7 +49,9 @@ public final class InsertAllRequest implements Serializable { /** * A Google Big Query row to be inserted into a table. Each {@code RowToInsert} has an associated - * id used by BigQuery to detect duplicate insertion requests on a best-effort basis. + * id used by BigQuery to detect duplicate insertion requests on a best-effort basis. Please + * notice that data for fields of type {@link Field.Type#bytes()} must be provided as a base64 + * encoded string. * *

Example usage of creating a row to insert: * 

 {@code
@@ -58,8 +60,9 @@ public final class InsertAllRequest implements Serializable {
    * recordContent.put("subfieldName1", "value");
    * recordContent.put("subfieldName2", repeatedFieldValue);
    * Map rowContent = new HashMap();
-   * rowContent.put("fieldName1", true);
-   * rowContent.put("fieldName2", recordContent);
+   * rowContent.put("booleanFieldName", true);
+   * rowContent.put("bytesFieldName", "DQ4KDQ==");
+   * rowContent.put("recordFieldName", recordContent);
    * RowToInsert row = new RowToInsert("rowId", rowContent);
    * }
* @@ -116,7 +119,8 @@ public boolean equals(Object obj) { } /** - * Creates a row to be inserted with associated id. + * Creates a row to be inserted with associated id. Please notice that data for fields of type + * {@link Field.Type#bytes()} must be provided as a base64 encoded string. * * @param id id of the row, used to identify duplicates * @param content the actual content of the row @@ -126,7 +130,8 @@ public static RowToInsert of(String id, Map content) { } /** - * Creates a row to be inserted without associated id. + * Creates a row to be inserted without associated id. Please notice that data for fields of + * type {@link Field.Type#bytes()} must be provided as a base64 encoded string. * * @param content the actual content of the row */ @@ -174,7 +179,8 @@ public Builder addRow(RowToInsert rowToInsert) { } /** - * Adds a row to be inserted with associated id. + * Adds a row to be inserted with associated id. Please notice that data for fields of type + * {@link Field.Type#bytes()} must be provided as a base64 encoded string. * *

Example usage of adding a row with associated id: * 

 {@code
@@ -184,8 +190,9 @@ public Builder addRow(RowToInsert rowToInsert) {
      * recordContent.put("subfieldName1", "value");
      * recordContent.put("subfieldName2", repeatedFieldValue);
      * Map rowContent = new HashMap();
-     * rowContent.put("fieldName1", true);
-     * rowContent.put("fieldName2", recordContent);
+     * rowContent.put("booleanFieldName", true);
+     * rowContent.put("bytesFieldName", "DQ4KDQ==");
+     * rowContent.put("recordFieldName", recordContent);
      * builder.addRow("rowId", rowContent);
      * }
*/ @@ -195,7 +202,8 @@ public Builder addRow(String id, Map content) { } /** - * Adds a row to be inserted without an associated id. + * Adds a row to be inserted without an associated id. Please notice that data for fields of + * type {@link Field.Type#bytes()} must be provided as a base64 encoded string. * *

Example usage of adding a row without an associated id: * 

 {@code
@@ -205,8 +213,9 @@ public Builder addRow(String id, Map content) {
      * recordContent.put("subfieldName1", "value");
      * recordContent.put("subfieldName2", repeatedFieldValue);
      * Map rowContent = new HashMap();
-     * rowContent.put("fieldName1", true);
-     * rowContent.put("fieldName2", recordContent);
+     * rowContent.put("booleanFieldName", true);
+     * rowContent.put("bytesFieldName", "DQ4KDQ==");
+     * rowContent.put("recordFieldName", recordContent);
      * builder.addRow(rowContent);
      * }
*/ diff --git a/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java b/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java index 03e2d7fea05e..ba55d9f67414 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java +++ b/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java @@ -21,6 +21,7 @@ import com.google.api.services.bigquery.model.JobConfigurationLoad; import com.google.common.base.MoreObjects.ToStringHelper; import com.google.common.collect.ImmutableList; +import com.google.common.primitives.Ints; import java.util.List; import java.util.Objects; @@ -97,12 +98,18 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur || loadConfigurationPb.getQuote() != null || loadConfigurationPb.getSkipLeadingRows() != null) { CsvOptions.Builder builder = CsvOptions.builder() - .allowJaggedRows(loadConfigurationPb.getAllowJaggedRows()) - .allowQuotedNewLines(loadConfigurationPb.getAllowQuotedNewlines()) .encoding(loadConfigurationPb.getEncoding()) .fieldDelimiter(loadConfigurationPb.getFieldDelimiter()) - .quote(loadConfigurationPb.getQuote()) - .skipLeadingRows(loadConfigurationPb.getSkipLeadingRows()); + .quote(loadConfigurationPb.getQuote()); + if (loadConfigurationPb.getAllowJaggedRows() != null) { + builder.allowJaggedRows(loadConfigurationPb.getAllowJaggedRows()); + } + if (loadConfigurationPb.getAllowQuotedNewlines() != null) { + builder.allowQuotedNewLines(loadConfigurationPb.getAllowQuotedNewlines()); + } + if (loadConfigurationPb.getSkipLeadingRows() != null) { + builder.skipLeadingRows(loadConfigurationPb.getSkipLeadingRows()); + } this.formatOptions = builder.build(); } this.maxBadRecords = loadConfigurationPb.getMaxBadRecords(); @@ -300,8 +307,11 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() { .setAllowJaggedRows(csvOptions.allowJaggedRows()) .setAllowQuotedNewlines(csvOptions.allowQuotedNewLines()) .setEncoding(csvOptions.encoding()) - .setQuote(csvOptions.quote()) - .setSkipLeadingRows(csvOptions.skipLeadingRows()); + .setQuote(csvOptions.quote()); + if (csvOptions.skipLeadingRows() != null) { + // todo(mziccard) remove checked cast or comment when #1044 is closed + loadConfigurationPb.setSkipLeadingRows(Ints.checkedCast(csvOptions.skipLeadingRows())); + } } if (schema != null) { loadConfigurationPb.setSchema(schema.toPb()); diff --git a/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java b/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java index 898063e7e0ed..354b4c9443db 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java +++ b/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java @@ -23,6 +23,7 @@ import com.google.cloud.bigquery.JobInfo.WriteDisposition; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; +import com.google.common.primitives.Ints; import java.io.Serializable; import java.util.List; @@ -90,12 +91,18 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur || loadConfigurationPb.getQuote() != null || loadConfigurationPb.getSkipLeadingRows() != null) { CsvOptions.Builder builder = CsvOptions.builder() - .allowJaggedRows(loadConfigurationPb.getAllowJaggedRows()) - .allowQuotedNewLines(loadConfigurationPb.getAllowQuotedNewlines()) .encoding(loadConfigurationPb.getEncoding()) .fieldDelimiter(loadConfigurationPb.getFieldDelimiter()) - .quote(loadConfigurationPb.getQuote()) - .skipLeadingRows(loadConfigurationPb.getSkipLeadingRows()); + .quote(loadConfigurationPb.getQuote()); + if (loadConfigurationPb.getAllowJaggedRows() != null) { + builder.allowJaggedRows(loadConfigurationPb.getAllowJaggedRows()); + } + if (loadConfigurationPb.getAllowQuotedNewlines() != null) { + builder.allowQuotedNewLines(loadConfigurationPb.getAllowQuotedNewlines()); + } + if (loadConfigurationPb.getSkipLeadingRows() != null) { + builder.skipLeadingRows(loadConfigurationPb.getSkipLeadingRows()); + } this.formatOptions = builder.build(); } this.maxBadRecords = loadConfigurationPb.getMaxBadRecords(); @@ -271,8 +278,11 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() { .setAllowJaggedRows(csvOptions.allowJaggedRows()) .setAllowQuotedNewlines(csvOptions.allowQuotedNewLines()) .setEncoding(csvOptions.encoding()) - .setQuote(csvOptions.quote()) - .setSkipLeadingRows(csvOptions.skipLeadingRows()); + .setQuote(csvOptions.quote()); + if (csvOptions.skipLeadingRows() != null) { + // todo(mziccard) remove checked cast or comment when #1044 is closed + loadConfigurationPb.setSkipLeadingRows(Ints.checkedCast(csvOptions.skipLeadingRows())); + } } if (schema != null) { loadConfigurationPb.setSchema(schema.toPb()); diff --git a/gcloud-java-bigquery/src/test/java/com/google/cloud/bigquery/CsvOptionsTest.java b/gcloud-java-bigquery/src/test/java/com/google/cloud/bigquery/CsvOptionsTest.java index df56a5ae096e..edc5e8508188 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/cloud/bigquery/CsvOptionsTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/cloud/bigquery/CsvOptionsTest.java @@ -30,7 +30,7 @@ public class CsvOptionsTest { private static final Charset ENCODING = StandardCharsets.UTF_8; private static final String FIELD_DELIMITER = ","; private static final String QUOTE = "\""; - private static final Integer SKIP_LEADING_ROWS = 42; + private static final long SKIP_LEADING_ROWS = 42L; private static final CsvOptions CSV_OPTIONS = CsvOptions.builder() .allowJaggedRows(ALLOW_JAGGED_ROWS) .allowQuotedNewLines(ALLOW_QUOTED_NEWLINE) @@ -65,7 +65,7 @@ public void testBuilder() { assertEquals(ENCODING.name(), CSV_OPTIONS.encoding()); assertEquals(FIELD_DELIMITER, CSV_OPTIONS.fieldDelimiter()); assertEquals(QUOTE, CSV_OPTIONS.quote()); - assertEquals(SKIP_LEADING_ROWS, CSV_OPTIONS.skipLeadingRows()); + assertEquals(SKIP_LEADING_ROWS, (long) CSV_OPTIONS.skipLeadingRows()); } @Test diff --git a/gcloud-java-bigquery/src/test/java/com/google/cloud/bigquery/FieldValueTest.java b/gcloud-java-bigquery/src/test/java/com/google/cloud/bigquery/FieldValueTest.java index 82086768b8ce..e104ab909f5b 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/cloud/bigquery/FieldValueTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/cloud/bigquery/FieldValueTest.java @@ -16,6 +16,7 @@ package com.google.cloud.bigquery; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; @@ -24,6 +25,7 @@ import com.google.api.services.bigquery.model.TableCell; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.io.BaseEncoding; import org.junit.Test; @@ -31,11 +33,14 @@ public class FieldValueTest { + private static final byte[] BYTES = {0xD, 0xE, 0xA, 0xD}; + private static final String BYTES_BASE64 = BaseEncoding.base64().encode(BYTES); private static final TableCell BOOLEAN_FIELD = new TableCell().setV("false"); private static final Map INTEGER_FIELD = ImmutableMap.of("v", "1"); private static final Map FLOAT_FIELD = ImmutableMap.of("v", "1.5"); private static final Map STRING_FIELD = ImmutableMap.of("v", "string"); private static final Map TIMESTAMP_FIELD = ImmutableMap.of("v", "42"); + private static final Map BYTES_FIELD = ImmutableMap.of("v", BYTES_BASE64); private static final Map NULL_FIELD = ImmutableMap.of("v", Data.nullOf(String.class)); private static final Map REPEATED_FIELD = @@ -60,6 +65,9 @@ public void testFromPb() { value = FieldValue.fromPb(TIMESTAMP_FIELD); assertEquals(FieldValue.Attribute.PRIMITIVE, value.attribute()); assertEquals(42000000, value.timestampValue()); + value = FieldValue.fromPb(BYTES_FIELD); + assertEquals(FieldValue.Attribute.PRIMITIVE, value.attribute()); + assertArrayEquals(BYTES, value.bytesValue()); value = FieldValue.fromPb(NULL_FIELD); assertNull(value.value()); value = FieldValue.fromPb(REPEATED_FIELD); @@ -94,6 +102,10 @@ public void testEquals() { assertEquals(timestampValue, FieldValue.fromPb(TIMESTAMP_FIELD)); assertEquals(timestampValue.hashCode(), FieldValue.fromPb(TIMESTAMP_FIELD).hashCode()); + FieldValue bytesValue = new FieldValue(FieldValue.Attribute.PRIMITIVE, BYTES_BASE64); + assertEquals(bytesValue, FieldValue.fromPb(BYTES_FIELD)); + assertEquals(bytesValue.hashCode(), FieldValue.fromPb(BYTES_FIELD).hashCode()); + FieldValue nullValue = new FieldValue(FieldValue.Attribute.PRIMITIVE, null); assertEquals(nullValue, FieldValue.fromPb(NULL_FIELD)); assertEquals(nullValue.hashCode(), FieldValue.fromPb(NULL_FIELD).hashCode()); diff --git a/gcloud-java-bigquery/src/test/java/com/google/cloud/bigquery/SerializationTest.java b/gcloud-java-bigquery/src/test/java/com/google/cloud/bigquery/SerializationTest.java index 30b1b9e067ec..8e12e3a6839d 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/cloud/bigquery/SerializationTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/cloud/bigquery/SerializationTest.java @@ -68,7 +68,7 @@ public class SerializationTest extends BaseSerializationTest { .encoding(StandardCharsets.ISO_8859_1) .fieldDelimiter(",") .quote("\"") - .skipLeadingRows(42) + .skipLeadingRows(42L) .build(); private static final Field FIELD_SCHEMA1 = Field.builder("StringField", Field.Type.string()) diff --git a/gcloud-java-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/gcloud-java-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index dde170f87859..5573b36d4cbc 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -16,6 +16,7 @@ package com.google.cloud.bigquery.it; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -68,6 +69,7 @@ import com.google.cloud.storage.testing.RemoteStorageHelper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.io.BaseEncoding; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -88,6 +90,8 @@ public class ITBigQueryTest { + private static final byte[] BYTES = {0xD, 0xE, 0xA, 0xD}; + private static final String BYTES_BASE64 = BaseEncoding.base64().encode(BYTES); private static final Logger LOG = Logger.getLogger(ITBigQueryTest.class.getName()); private static final String DATASET = RemoteBigQueryHelper.generateDatasetName(); private static final String DESCRIPTION = "Test dataset"; @@ -112,14 +116,19 @@ public class ITBigQueryTest { .mode(Field.Mode.NULLABLE) .description("BooleanDescription") .build(); + private static final Field BYTES_FIELD_SCHEMA = + Field.builder("BytesField", Field.Type.bytes()) + .mode(Field.Mode.NULLABLE) + .description("BytesDescription") + .build(); private static final Field RECORD_FIELD_SCHEMA = Field.builder("RecordField", Field.Type.record(TIMESTAMP_FIELD_SCHEMA, - STRING_FIELD_SCHEMA, INTEGER_FIELD_SCHEMA, BOOLEAN_FIELD_SCHEMA)) + STRING_FIELD_SCHEMA, INTEGER_FIELD_SCHEMA, BOOLEAN_FIELD_SCHEMA, BYTES_FIELD_SCHEMA)) .mode(Field.Mode.REQUIRED) .description("RecordDescription") .build(); private static final Schema TABLE_SCHEMA = Schema.of(TIMESTAMP_FIELD_SCHEMA, STRING_FIELD_SCHEMA, - INTEGER_FIELD_SCHEMA, BOOLEAN_FIELD_SCHEMA, RECORD_FIELD_SCHEMA); + INTEGER_FIELD_SCHEMA, BOOLEAN_FIELD_SCHEMA, BYTES_FIELD_SCHEMA, RECORD_FIELD_SCHEMA); private static final Schema SIMPLE_SCHEMA = Schema.of(STRING_FIELD_SCHEMA); private static final Schema QUERY_RESULT_SCHEMA = Schema.builder() .addField(Field.builder("TimestampField", Field.Type.timestamp()) @@ -143,11 +152,13 @@ public class ITBigQueryTest { + "\"StringField\": \"stringValue\"," + "\"IntegerField\": [\"0\", \"1\"]," + "\"BooleanField\": \"false\"," + + "\"BytesField\": \"" + BYTES_BASE64 + "\"," + "\"RecordField\": {" + "\"TimestampField\": \"1969-07-20 20:18:04 UTC\"," + "\"StringField\": null," + "\"IntegerField\": [\"1\",\"0\"]," - + "\"BooleanField\": \"true\"" + + "\"BooleanField\": \"true\"," + + "\"BytesField\": \"" + BYTES_BASE64 + "\"" + "}" + "}\n" + "{" @@ -155,11 +166,13 @@ public class ITBigQueryTest { + "\"StringField\": \"stringValue\"," + "\"IntegerField\": [\"0\", \"1\"]," + "\"BooleanField\": \"false\"," + + "\"BytesField\": \"" + BYTES_BASE64 + "\"," + "\"RecordField\": {" + "\"TimestampField\": \"1969-07-20 20:18:04 UTC\"," + "\"StringField\": null," + "\"IntegerField\": [\"1\",\"0\"]," - + "\"BooleanField\": \"true\"" + + "\"BooleanField\": \"true\"," + + "\"BytesField\": \"" + BYTES_BASE64 + "\"" + "}" + "}"; @@ -510,30 +523,36 @@ public void testDeleteNonExistingTable() { } @Test - public void testInsertAll() { + public void testInsertAll() throws IOException { String tableName = "test_insert_all_table"; StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA); TableInfo tableInfo = TableInfo.of(TableId.of(DATASET, tableName), tableDefinition); assertNotNull(bigquery.create(tableInfo)); + ImmutableMap.Builder builder1 = ImmutableMap.builder(); + builder1.put("TimestampField", "2014-08-19 07:41:35.220 -05:00"); + builder1.put("StringField", "stringValue"); + builder1.put("IntegerField", ImmutableList.of(0, 1)); + builder1.put("BooleanField", false); + builder1.put("BytesField", BYTES_BASE64); + builder1.put("RecordField", ImmutableMap.of( + "TimestampField", "1969-07-20 20:18:04 UTC", + "IntegerField", ImmutableList.of(1, 0), + "BooleanField", true, + "BytesField", BYTES_BASE64)); + ImmutableMap.Builder builder2 = ImmutableMap.builder(); + builder2.put("TimestampField", "2014-08-19 07:41:35.220 -05:00"); + builder2.put("StringField", "stringValue"); + builder2.put("IntegerField", ImmutableList.of(0, 1)); + builder2.put("BooleanField", false); + builder2.put("BytesField", BYTES_BASE64); + builder2.put("RecordField", ImmutableMap.of( + "TimestampField", "1969-07-20 20:18:04 UTC", + "IntegerField", ImmutableList.of(1, 0), + "BooleanField", true, + "BytesField", BYTES_BASE64)); InsertAllRequest request = InsertAllRequest.builder(tableInfo.tableId()) - .addRow(ImmutableMap.of( - "TimestampField", "2014-08-19 07:41:35.220 -05:00", - "StringField", "stringValue", - "IntegerField", ImmutableList.of(0, 1), - "BooleanField", false, - "RecordField", ImmutableMap.of( - "TimestampField", "1969-07-20 20:18:04 UTC", - "IntegerField", ImmutableList.of(1, 0), - "BooleanField", true))) - .addRow(ImmutableMap.of( - "TimestampField", "2014-08-19 07:41:35.220 -05:00", - "StringField", "stringValue", - "IntegerField", ImmutableList.of(0, 1), - "BooleanField", false, - "RecordField", ImmutableMap.of( - "TimestampField", "1969-07-20 20:18:04 UTC", - "IntegerField", ImmutableList.of(1, 0), - "BooleanField", true))) + .addRow(builder1.build()) + .addRow(builder2.build()) .build(); InsertAllResponse response = bigquery.insertAll(request); assertFalse(response.hasErrors()); @@ -547,25 +566,31 @@ public void testInsertAllWithSuffix() throws InterruptedException { StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA); TableInfo tableInfo = TableInfo.of(TableId.of(DATASET, tableName), tableDefinition); assertNotNull(bigquery.create(tableInfo)); + ImmutableMap.Builder builder1 = ImmutableMap.builder(); + builder1.put("TimestampField", "2014-08-19 07:41:35.220 -05:00"); + builder1.put("StringField", "stringValue"); + builder1.put("IntegerField", ImmutableList.of(0, 1)); + builder1.put("BooleanField", false); + builder1.put("BytesField", BYTES_BASE64); + builder1.put("RecordField", ImmutableMap.of( + "TimestampField", "1969-07-20 20:18:04 UTC", + "IntegerField", ImmutableList.of(1, 0), + "BooleanField", true, + "BytesField", BYTES_BASE64)); + ImmutableMap.Builder builder2 = ImmutableMap.builder(); + builder2.put("TimestampField", "2014-08-19 07:41:35.220 -05:00"); + builder2.put("StringField", "stringValue"); + builder2.put("IntegerField", ImmutableList.of(0, 1)); + builder2.put("BooleanField", false); + builder2.put("BytesField", BYTES_BASE64); + builder2.put("RecordField", ImmutableMap.of( + "TimestampField", "1969-07-20 20:18:04 UTC", + "IntegerField", ImmutableList.of(1, 0), + "BooleanField", true, + "BytesField", BYTES_BASE64)); InsertAllRequest request = InsertAllRequest.builder(tableInfo.tableId()) - .addRow(ImmutableMap.of( - "TimestampField", "2014-08-19 07:41:35.220 -05:00", - "StringField", "stringValue", - "IntegerField", ImmutableList.of(0, 1), - "BooleanField", false, - "RecordField", ImmutableMap.of( - "TimestampField", "1969-07-20 20:18:04 UTC", - "IntegerField", ImmutableList.of(1, 0), - "BooleanField", true))) - .addRow(ImmutableMap.of( - "TimestampField", "2014-08-19 07:41:35.220 -05:00", - "StringField", "stringValue", - "IntegerField", ImmutableList.of(0, 1), - "BooleanField", false, - "RecordField", ImmutableMap.of( - "TimestampField", "1969-07-20 20:18:04 UTC", - "IntegerField", ImmutableList.of(1, 0), - "BooleanField", true))) + .addRow(builder1.build()) + .addRow(builder2.build()) .templateSuffix("_suffix") .build(); InsertAllResponse response = bigquery.insertAll(request); @@ -588,30 +613,38 @@ public void testInsertAllWithErrors() { StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA); TableInfo tableInfo = TableInfo.of(TableId.of(DATASET, tableName), tableDefinition); assertNotNull(bigquery.create(tableInfo)); + ImmutableMap.Builder builder1 = ImmutableMap.builder(); + builder1.put("TimestampField", "2014-08-19 07:41:35.220 -05:00"); + builder1.put("StringField", "stringValue"); + builder1.put("IntegerField", ImmutableList.of(0, 1)); + builder1.put("BooleanField", false); + builder1.put("BytesField", BYTES_BASE64); + builder1.put("RecordField", ImmutableMap.of( + "TimestampField", "1969-07-20 20:18:04 UTC", + "IntegerField", ImmutableList.of(1, 0), + "BooleanField", true, + "BytesField", BYTES_BASE64)); + ImmutableMap.Builder builder2 = ImmutableMap.builder(); + builder2.put("TimestampField", "invalidDate"); + builder2.put("StringField", "stringValue"); + builder2.put("IntegerField", ImmutableList.of(0, 1)); + builder2.put("BooleanField", false); + builder2.put("BytesField", BYTES_BASE64); + builder2.put("RecordField", ImmutableMap.of( + "TimestampField", "1969-07-20 20:18:04 UTC", + "IntegerField", ImmutableList.of(1, 0), + "BooleanField", true, + "BytesField", BYTES_BASE64)); + ImmutableMap.Builder builder3 = ImmutableMap.builder(); + builder3.put("TimestampField", "2014-08-19 07:41:35.220 -05:00"); + builder3.put("StringField", "stringValue"); + builder3.put("IntegerField", ImmutableList.of(0, 1)); + builder3.put("BooleanField", false); + builder3.put("BytesField", BYTES_BASE64); InsertAllRequest request = InsertAllRequest.builder(tableInfo.tableId()) - .addRow(ImmutableMap.of( - "TimestampField", "2014-08-19 07:41:35.220 -05:00", - "StringField", "stringValue", - "IntegerField", ImmutableList.of(0, 1), - "BooleanField", false, - "RecordField", ImmutableMap.of( - "TimestampField", "1969-07-20 20:18:04 UTC", - "IntegerField", ImmutableList.of(1, 0), - "BooleanField", true))) - .addRow(ImmutableMap.of( - "TimestampField", "invalidDate", - "StringField", "stringValue", - "IntegerField", ImmutableList.of(0, 1), - "BooleanField", false, - "RecordField", ImmutableMap.of( - "TimestampField", "1969-07-20 20:18:04 UTC", - "IntegerField", ImmutableList.of(1, 0), - "BooleanField", true))) - .addRow(ImmutableMap.of( - "TimestampField", "1969-07-20 20:18:04 UTC", - "StringField", "stringValue", - "IntegerField", ImmutableList.of(0, 1), - "BooleanField", false)) + .addRow(builder1.build()) + .addRow(builder2.build()) + .addRow(builder3.build()) .skipInvalidRows(true) .build(); InsertAllResponse response = bigquery.insertAll(request); @@ -631,17 +664,20 @@ public void testListAllTableData() { FieldValue stringCell = row.get(1); FieldValue integerCell = row.get(2); FieldValue booleanCell = row.get(3); - FieldValue recordCell = row.get(4); + FieldValue bytesCell = row.get(4); + FieldValue recordCell = row.get(5); assertEquals(FieldValue.Attribute.PRIMITIVE, timestampCell.attribute()); assertEquals(FieldValue.Attribute.PRIMITIVE, stringCell.attribute()); assertEquals(FieldValue.Attribute.REPEATED, integerCell.attribute()); assertEquals(FieldValue.Attribute.PRIMITIVE, booleanCell.attribute()); + assertEquals(FieldValue.Attribute.PRIMITIVE, bytesCell.attribute()); assertEquals(FieldValue.Attribute.RECORD, recordCell.attribute()); assertEquals(1408452095220000L, timestampCell.timestampValue()); assertEquals("stringValue", stringCell.stringValue()); assertEquals(0, integerCell.repeatedValue().get(0).longValue()); assertEquals(1, integerCell.repeatedValue().get(1).longValue()); assertEquals(false, booleanCell.booleanValue()); + assertArrayEquals(BYTES, bytesCell.bytesValue()); assertEquals(-14182916000000L, recordCell.recordValue().get(0).timestampValue()); assertTrue(recordCell.recordValue().get(1).isNull()); assertEquals(1, recordCell.recordValue().get(2).repeatedValue().get(0).longValue()); @@ -920,17 +956,20 @@ public void testInsertFromFile() throws InterruptedException { FieldValue stringCell = row.get(1); FieldValue integerCell = row.get(2); FieldValue booleanCell = row.get(3); - FieldValue recordCell = row.get(4); + FieldValue bytesCell = row.get(4); + FieldValue recordCell = row.get(5); assertEquals(FieldValue.Attribute.PRIMITIVE, timestampCell.attribute()); assertEquals(FieldValue.Attribute.PRIMITIVE, stringCell.attribute()); assertEquals(FieldValue.Attribute.REPEATED, integerCell.attribute()); assertEquals(FieldValue.Attribute.PRIMITIVE, booleanCell.attribute()); + assertEquals(FieldValue.Attribute.PRIMITIVE, bytesCell.attribute()); assertEquals(FieldValue.Attribute.RECORD, recordCell.attribute()); assertEquals(1408452095220000L, timestampCell.timestampValue()); assertEquals("stringValue", stringCell.stringValue()); assertEquals(0, integerCell.repeatedValue().get(0).longValue()); assertEquals(1, integerCell.repeatedValue().get(1).longValue()); assertEquals(false, booleanCell.booleanValue()); + assertArrayEquals(BYTES, bytesCell.bytesValue()); assertEquals(-14182916000000L, recordCell.recordValue().get(0).timestampValue()); assertTrue(recordCell.recordValue().get(1).isNull()); assertEquals(1, recordCell.recordValue().get(2).repeatedValue().get(0).longValue()); diff --git a/gcloud-java-examples/src/main/java/com/google/cloud/examples/bigquery/BigQueryExample.java b/gcloud-java-examples/src/main/java/com/google/cloud/examples/bigquery/BigQueryExample.java index 156e475a5ac9..3ac1335cca7a 100644 --- a/gcloud-java-examples/src/main/java/com/google/cloud/examples/bigquery/BigQueryExample.java +++ b/gcloud-java-examples/src/main/java/com/google/cloud/examples/bigquery/BigQueryExample.java @@ -92,10 +92,10 @@ * operations that apply to more than one entity (`list`, `create`, `info` and `delete`) the third * parameter specifies the entity. {@code } indicates that only primitive types are * supported by the {@code create table} and {@code create external-table} operations - * ({@code string}, {@code float}, {@code integer}, {@code timestamp}, {@code boolean}). - * {@code }, {@code } and {@code } parameters are URIs to - * Google Cloud Storage blobs, in the form {@code gs://bucket/path}. See each action's run method - * for the specific BigQuery interaction. + * ({@code string}, {@code float}, {@code integer}, {@code timestamp}, {@code boolean}, + * {@code bytes}). {@code }, {@code } and {@code } + * parameters are URIs to Google Cloud Storage blobs, in the form {@code gs://bucket/path}. + * See each action's run method for the specific BigQuery interaction. */ public class BigQueryExample { @@ -426,6 +426,9 @@ static Schema parseSchema(String[] args, int start, int end) { case "boolean": fieldType = Field.Type.bool(); break; + case "bytes": + fieldType = Field.Type.bytes(); + break; default: throw new IllegalArgumentException("Unrecognized field type '" + typeString + "'."); }