From a262306eeff567748b252f0fd8e1771c7dbd5a4f Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Fri, 9 Sep 2016 16:42:51 +0200 Subject: [PATCH 1/2] Add support for customer-supplied encryption keys --- .../java/com/google/cloud/storage/Blob.java | 33 ++++ .../com/google/cloud/storage/BlobInfo.java | 89 ++++++++++ .../java/com/google/cloud/storage/Bucket.java | 46 +++++ .../com/google/cloud/storage/Storage.java | 63 ++++++- .../cloud/storage/spi/DefaultStorageRpc.java | 46 ++++- .../google/cloud/storage/spi/StorageRpc.java | 3 +- .../google/cloud/storage/BlobInfoTest.java | 25 +++ .../com/google/cloud/storage/BlobTest.java | 50 +++++- .../com/google/cloud/storage/BucketTest.java | 12 +- .../google/cloud/storage/StorageImplTest.java | 159 +++++++++++++++--- .../cloud/storage/it/ITStorageTest.java | 94 ++++++++++- 11 files changed, 579 insertions(+), 41 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java index 1c2f9f57243f..9ca4b1ad633f 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java @@ -36,10 +36,12 @@ import com.google.cloud.storage.spi.StorageRpc; import com.google.cloud.storage.spi.StorageRpc.Tuple; import com.google.common.base.Function; +import com.google.common.io.BaseEncoding; import java.io.IOException; import java.io.ObjectInputStream; import java.net.URL; +import java.security.Key; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -81,6 +83,10 @@ private BlobSourceOption(StorageRpc.Option rpcOption) { super(rpcOption, null); } + private BlobSourceOption(StorageRpc.Option rpcOption, Object value) { + super(rpcOption, value); + } + private Storage.BlobSourceOption toSourceOptions(BlobInfo blobInfo) { switch (rpcOption()) { case IF_GENERATION_MATCH: @@ -91,6 +97,8 @@ private Storage.BlobSourceOption toSourceOptions(BlobInfo blobInfo) { return Storage.BlobSourceOption.metagenerationMatch(blobInfo.metageneration()); case IF_METAGENERATION_NOT_MATCH: return Storage.BlobSourceOption.metagenerationNotMatch(blobInfo.metageneration()); + case CUSTOMER_SUPPLIED_KEY: + return Storage.BlobSourceOption.decryptionKey((String) value()); default: throw new AssertionError("Unexpected enum value"); } @@ -143,6 +151,25 @@ public static BlobSourceOption metagenerationNotMatch() { return new BlobSourceOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH); } + /** + * Returns an option to set a customer-supplied AES256 key for server-side encryption of the + * blob. + */ + public static BlobSourceOption decryptionKey(Key key) { + String base64Key = BaseEncoding.base64().encode(key.getEncoded()); + return new BlobSourceOption(StorageRpc.Option.CUSTOMER_SUPPLIED_KEY, base64Key); + } + + /** + * Returns an option to set a customer-supplied AES256 key for server-side encryption of the + * blob. + * + * @param key the AES256 encoded in base64 + */ + public static BlobSourceOption decryptionKey(String key) { + return new BlobSourceOption(StorageRpc.Option.CUSTOMER_SUPPLIED_KEY, key); + } + static Storage.BlobSourceOption[] toSourceOptions(BlobInfo blobInfo, BlobSourceOption... options) { Storage.BlobSourceOption[] convertedOptions = new Storage.BlobSourceOption[options.length]; @@ -308,6 +335,12 @@ Builder isDirectory(boolean isDirectory) { return this; } + @Override + Builder customerEncryption(CustomerEncryption customerEncryption) { + infoBuilder.customerEncryption(customerEncryption); + return this; + } + @Override public Blob build() { return new Blob(storage, infoBuilder); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java index 85404ff84b82..149e5d02e95d 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java @@ -80,6 +80,7 @@ public StorageObject apply(BlobInfo blobInfo) { private final String contentLanguage; private final Integer componentCount; private final boolean isDirectory; + private final CustomerEncryption customerEncryption; /** * This class is meant for internal use only. Users are discouraged from using this class. @@ -92,6 +93,69 @@ public Set> entrySet() { } } + /** + * Objects of this class hold information on the customer-supplied encryption key, if the blob is + * encrypted using such a key. + */ + public static class CustomerEncryption implements Serializable { + + private static final long serialVersionUID = -2133042982786959351L; + + private final String encryptionAlgorithm; + private final String keySha256; + + CustomerEncryption(String encryptionAlgorithm, String keySha256) { + this.encryptionAlgorithm = encryptionAlgorithm; + this.keySha256 = keySha256; + } + + /** + * Returns the algorithm used to encrypt the blob. + */ + public String encryptionAlgorithm() { + return encryptionAlgorithm; + } + + /** + * Returns the SHA256 hash of the encryption key. + */ + public String keySha256() { + return keySha256; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("encryptionAlgorithm", encryptionAlgorithm()) + .add("keySha256", keySha256()) + .toString(); + } + + @Override + public final int hashCode() { + return Objects.hash(encryptionAlgorithm, keySha256); + } + + @Override + public final boolean equals(Object obj) { + return obj == this + || obj != null + && obj.getClass().equals(CustomerEncryption.class) + && Objects.equals(toPb(), ((CustomerEncryption) obj).toPb()); + } + + StorageObject.CustomerEncryption toPb() { + return new StorageObject.CustomerEncryption() + .setEncryptionAlgorithm(encryptionAlgorithm) + .setKeySha256(keySha256); + } + + static CustomerEncryption fromPb(StorageObject.CustomerEncryption customerEncryptionPb) { + return new CustomerEncryption(customerEncryptionPb.getEncryptionAlgorithm(), + customerEncryptionPb.getKeySha256()); + } + } + /** * Builder for {@code BlobInfo}. */ @@ -193,6 +257,8 @@ public abstract static class Builder { abstract Builder isDirectory(boolean isDirectory); + abstract Builder customerEncryption(CustomerEncryption customerEncryption); + /** * Creates a {@code BlobInfo} object. */ @@ -223,6 +289,7 @@ static final class BuilderImpl extends Builder { private Long updateTime; private Long createTime; private Boolean isDirectory; + private CustomerEncryption customerEncryption; BuilderImpl(BlobId blobId) { this.blobId = blobId; @@ -237,6 +304,7 @@ static final class BuilderImpl extends Builder { contentDisposition = blobInfo.contentDisposition; contentLanguage = blobInfo.contentLanguage; componentCount = blobInfo.componentCount; + customerEncryption = blobInfo.customerEncryption; acl = blobInfo.acl; owner = blobInfo.owner; size = blobInfo.size; @@ -386,6 +454,12 @@ Builder isDirectory(boolean isDirectory) { return this; } + @Override + Builder customerEncryption(CustomerEncryption customerEncryption) { + this.customerEncryption = customerEncryption; + return this; + } + @Override public BlobInfo build() { checkNotNull(blobId); @@ -402,6 +476,7 @@ public BlobInfo build() { contentDisposition = builder.contentDisposition; contentLanguage = builder.contentLanguage; componentCount = builder.componentCount; + customerEncryption = builder.customerEncryption; acl = builder.acl; owner = builder.owner; size = builder.size; @@ -631,6 +706,14 @@ public boolean isDirectory() { return isDirectory; } + /** + * Returns information on the customer-supplied encryption key, if the blob is encrypted using + * such a key. + */ + public CustomerEncryption customerEncryption() { + return customerEncryption; + } + /** * Returns a builder for the current blob. */ @@ -696,6 +779,9 @@ public ObjectAccessControl apply(Acl acl) { firstNonNull(entry.getValue(), Data.nullOf(String.class))); } } + if (customerEncryption != null) { + storageObject.setCustomerEncryption(customerEncryption.toPb()); + } storageObject.setMetadata(pbMetadata); storageObject.setCacheControl(cacheControl); storageObject.setContentEncoding(contentEncoding); @@ -815,6 +901,9 @@ public Acl apply(ObjectAccessControl objectAccessControl) { if (storageObject.containsKey("isDirectory")) { builder.isDirectory(Boolean.TRUE); } + if (storageObject.getCustomerEncryption() != null) { + builder.customerEncryption(CustomerEncryption.fromPb(storageObject.getCustomerEncryption())); + } return builder.build(); } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java index 9955dda330f3..7377e1742135 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java @@ -31,11 +31,13 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import com.google.common.io.BaseEncoding; import java.io.IOException; import java.io.InputStream; import java.io.ObjectInputStream; import java.io.Serializable; +import java.security.Key; import java.util.Arrays; import java.util.List; import java.util.Objects; @@ -166,6 +168,9 @@ private StorageRpc.Tuple toTargetOption(Blob case IF_METAGENERATION_NOT_MATCH: return StorageRpc.Tuple.of(blobInfo.toBuilder().metageneration((Long) value()).build(), Storage.BlobTargetOption.metagenerationNotMatch()); + case CUSTOMER_SUPPLIED_KEY: + return StorageRpc.Tuple.of(blobInfo, + Storage.BlobTargetOption.encryptionKey((String) value())); default: throw new AssertionError("Unexpected enum value"); } @@ -223,6 +228,25 @@ public static BlobTargetOption metagenerationNotMatch(long metageneration) { return new BlobTargetOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH, metageneration); } + /** + * Returns an option to set a customer-supplied AES256 key for server-side encryption of the + * blob. + */ + public static BlobTargetOption encryptionKey(Key key) { + String base64Key = BaseEncoding.base64().encode(key.getEncoded()); + return new BlobTargetOption(StorageRpc.Option.CUSTOMER_SUPPLIED_KEY, base64Key); + } + + /** + * Returns an option to set a customer-supplied AES256 key for server-side encryption of the + * blob. + * + * @param key the AES256 encoded in base64 + */ + public static BlobTargetOption encryptionKey(String key) { + return new BlobTargetOption(StorageRpc.Option.CUSTOMER_SUPPLIED_KEY, key); + } + static StorageRpc.Tuple toTargetOptions( BlobInfo info, BlobTargetOption... options) { Set optionSet = @@ -289,6 +313,9 @@ private StorageRpc.Tuple toWriteOption(BlobIn case IF_CRC32C_MATCH: return StorageRpc.Tuple.of(blobInfo.toBuilder().crc32c((String) value).build(), Storage.BlobWriteOption.crc32cMatch()); + case CUSTOMER_SUPPLIED_KEY: + return StorageRpc.Tuple.of(blobInfo, + Storage.BlobWriteOption.encryptionKey((String) value)); default: throw new AssertionError("Unexpected enum value"); } @@ -387,6 +414,25 @@ public static BlobWriteOption crc32cMatch(String crc32c) { return new BlobWriteOption(Storage.BlobWriteOption.Option.IF_CRC32C_MATCH, crc32c); } + /** + * Returns an option to set a customer-supplied AES256 key for server-side encryption of the + * blob. + */ + public static BlobWriteOption encryptionKey(Key key) { + String base64Key = BaseEncoding.base64().encode(key.getEncoded()); + return new BlobWriteOption(Storage.BlobWriteOption.Option.CUSTOMER_SUPPLIED_KEY, base64Key); + } + + /** + * Returns an option to set a customer-supplied AES256 key for server-side encryption of the + * blob. + * + * @param key the AES256 encoded in base64 + */ + public static BlobWriteOption encryptionKey(String key) { + return new BlobWriteOption(Storage.BlobWriteOption.Option.CUSTOMER_SUPPLIED_KEY, key); + } + static StorageRpc.Tuple toWriteOptions( BlobInfo info, BlobWriteOption... options) { Set optionSet = diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 4d8f44608242..08202202ee64 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -36,10 +36,12 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import com.google.common.io.BaseEncoding; import java.io.InputStream; import java.io.Serializable; import java.net.URL; +import java.security.Key; import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashSet; @@ -325,6 +327,25 @@ public static BlobTargetOption metagenerationNotMatch() { return new BlobTargetOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH); } + /** + * Returns an option to set a customer-supplied AES256 key for server-side encryption of the + * blob. + */ + public static BlobTargetOption encryptionKey(Key key) { + String base64Key = BaseEncoding.base64().encode(key.getEncoded()); + return new BlobTargetOption(StorageRpc.Option.CUSTOMER_SUPPLIED_KEY, base64Key); + } + + /** + * Returns an option to set a customer-supplied AES256 key for server-side encryption of the + * blob. + * + * @param key the AES256 encoded in base64 + */ + public static BlobTargetOption encryptionKey(String key) { + return new BlobTargetOption(StorageRpc.Option.CUSTOMER_SUPPLIED_KEY, key); + } + static Tuple convert(BlobInfo info, BlobWriteOption... options) { BlobInfo.Builder infoBuilder = info.toBuilder().crc32c(null).md5(null); List targetOptions = Lists.newArrayListWithCapacity(options.length); @@ -358,7 +379,7 @@ class BlobWriteOption implements Serializable { enum Option { PREDEFINED_ACL, IF_GENERATION_MATCH, IF_GENERATION_NOT_MATCH, IF_METAGENERATION_MATCH, - IF_METAGENERATION_NOT_MATCH, IF_MD5_MATCH, IF_CRC32C_MATCH; + IF_METAGENERATION_NOT_MATCH, IF_MD5_MATCH, IF_CRC32C_MATCH, CUSTOMER_SUPPLIED_KEY; StorageRpc.Option toRpcOption() { return StorageRpc.Option.valueOf(this.name()); @@ -456,6 +477,25 @@ public static BlobWriteOption md5Match() { public static BlobWriteOption crc32cMatch() { return new BlobWriteOption(Option.IF_CRC32C_MATCH, true); } + + /** + * Returns an option to set a customer-supplied AES256 key for server-side encryption of the + * blob. + */ + public static BlobWriteOption encryptionKey(Key key) { + String base64Key = BaseEncoding.base64().encode(key.getEncoded()); + return new BlobWriteOption(Option.CUSTOMER_SUPPLIED_KEY, base64Key); + } + + /** + * Returns an option to set a customer-supplied AES256 key for server-side encryption of the + * blob. + * + * @param key the AES256 encoded in base64 + */ + public static BlobWriteOption encryptionKey(String key) { + return new BlobWriteOption(Option.CUSTOMER_SUPPLIED_KEY, key); + } } /** @@ -465,7 +505,7 @@ class BlobSourceOption extends Option { private static final long serialVersionUID = -3712768261070182991L; - private BlobSourceOption(StorageRpc.Option rpcOption, Long value) { + private BlobSourceOption(StorageRpc.Option rpcOption, Object value) { super(rpcOption, value); } @@ -522,6 +562,25 @@ public static BlobSourceOption metagenerationMatch(long metageneration) { public static BlobSourceOption metagenerationNotMatch(long metageneration) { return new BlobSourceOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH, metageneration); } + + /** + * Returns an option to set a customer-supplied AES256 key for server-side encryption of the + * blob. + */ + public static BlobSourceOption decryptionKey(Key key) { + String base64Key = BaseEncoding.base64().encode(key.getEncoded()); + return new BlobSourceOption(StorageRpc.Option.CUSTOMER_SUPPLIED_KEY, base64Key); + } + + /** + * Returns an option to set a customer-supplied AES256 key for server-side encryption of the + * blob. + * + * @param key the AES256 encoded in base64 + */ + public static BlobSourceOption decryptionKey(String key) { + return new BlobSourceOption(StorageRpc.Option.CUSTOMER_SUPPLIED_KEY, key); + } } /** diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/DefaultStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/DefaultStorageRpc.java index 2e77b511d554..5620b333ed34 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/DefaultStorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/DefaultStorageRpc.java @@ -16,6 +16,7 @@ package com.google.cloud.storage.spi; +import static com.google.cloud.storage.spi.StorageRpc.Option.CUSTOMER_SUPPLIED_KEY; import static com.google.cloud.storage.spi.StorageRpc.Option.DELIMITER; import static com.google.cloud.storage.spi.StorageRpc.Option.FIELDS; import static com.google.cloud.storage.spi.StorageRpc.Option.IF_GENERATION_MATCH; @@ -70,6 +71,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hashing; +import com.google.common.io.BaseEncoding; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -83,6 +87,8 @@ public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; + private static final String ENCRYPTION_KEY_PREFIX = "x-goog-encryption-"; + private static final String SOURCE_ENCRYPTION_KEY_PREFIX = "x-goog-copy-source-encryption-"; private final StorageOptions options; private final Storage storage; @@ -222,6 +228,19 @@ private static StorageException translate(GoogleJsonError exception) { return new StorageException(exception); } + private static void setEncryptionHeaders(HttpHeaders headers, String headerPrefix, + Map options) { + String key = CUSTOMER_SUPPLIED_KEY.getString(options); + if (key != null) { + BaseEncoding base64 = BaseEncoding.base64(); + HashFunction hashFunction = Hashing.sha256(); + headers.set(headerPrefix + "algorithm", "AES256"); + headers.set(headerPrefix + "key", key); + headers.set(headerPrefix + "key-sha256", + base64.encode(hashFunction.hashBytes(base64.decode(key)).asBytes())); + } + } + @Override public Bucket create(Bucket bucket, Map options) { try { @@ -244,6 +263,7 @@ public StorageObject create(StorageObject storageObject, final InputStream conte .insert(storageObject.getBucket(), storageObject, new InputStreamContent(storageObject.getContentType(), content)); insert.getMediaHttpUploader().setDirectUploadEnabled(true); + setEncryptionHeaders(insert.getRequestHeaders(), ENCRYPTION_KEY_PREFIX, options); return insert.setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) @@ -474,6 +494,7 @@ public byte[] load(StorageObject from, Map options) { .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); + setEncryptionHeaders(getRequest.getRequestHeaders(), ENCRYPTION_KEY_PREFIX, options); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.executeMedia().download(out); return out.toByteArray(); @@ -501,7 +522,9 @@ public Tuple read(StorageObject from, Map options, lo checkArgument(position >= 0, "Position should be non-negative, is %d", position); StringBuilder range = new StringBuilder(); range.append("bytes=").append(position).append("-").append(position + bytes - 1); - req.getRequestHeaders().setRange(range.toString()); + HttpHeaders requestHeaders = req.getRequestHeaders(); + requestHeaders.setRange(range.toString()); + setEncryptionHeaders(requestHeaders, ENCRYPTION_KEY_PREFIX, options); ByteArrayOutputStream output = new ByteArrayOutputStream(bytes); req.executeMedia().download(output); String etag = req.getLastResponseHeaders().getETag(); @@ -586,8 +609,18 @@ public String open(StorageObject object, Map options) { HttpRequestFactory requestFactory = storage.getRequestFactory(); HttpRequest httpRequest = requestFactory.buildPostRequest(url, new JsonHttpContent(jsonFactory, object)); - httpRequest.getHeaders().set("X-Upload-Content-Type", + HttpHeaders requestHeaders = httpRequest.getHeaders(); + requestHeaders.set("X-Upload-Content-Type", firstNonNull(object.getContentType(), "application/octet-stream")); + String key = CUSTOMER_SUPPLIED_KEY.getString(options); + if (key != null) { + BaseEncoding base64 = BaseEncoding.base64(); + HashFunction hashFunction = Hashing.sha256(); + requestHeaders.set("x-goog-encryption-algorithm", "AES256"); + requestHeaders.set("x-goog-encryption-key", key); + requestHeaders.set("x-goog-encryption-key-sha256", + base64.encode(hashFunction.hashBytes(base64.decode(key)).asBytes())); + } HttpResponse response = httpRequest.execute(); if (response.getStatusCode() != 200) { GoogleJsonError error = new GoogleJsonError(); @@ -615,7 +648,7 @@ private RewriteResponse rewrite(RewriteRequest req, String token) { try { Long maxBytesRewrittenPerCall = req.megabytesRewrittenPerCall != null ? req.megabytesRewrittenPerCall * MEGABYTE : null; - com.google.api.services.storage.model.RewriteResponse rewriteResponse = storage.objects() + Storage.Objects.Rewrite rewrite = storage.objects() .rewrite(req.source.getBucket(), req.source.getName(), req.target.getBucket(), req.target.getName(), req.overrideInfo ? req.target : null) .setSourceGeneration(req.source.getGeneration()) @@ -630,8 +663,11 @@ private RewriteResponse rewrite(RewriteRequest req, String token) { .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(req.targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(req.targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(req.targetOptions)) - .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(req.targetOptions)) - .execute(); + .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(req.targetOptions)); + HttpHeaders requestHeaders = rewrite.getRequestHeaders(); + setEncryptionHeaders(requestHeaders, SOURCE_ENCRYPTION_KEY_PREFIX, req.sourceOptions); + setEncryptionHeaders(requestHeaders, ENCRYPTION_KEY_PREFIX, req.targetOptions); + com.google.api.services.storage.model.RewriteResponse rewriteResponse = rewrite.execute(); return new RewriteResponse( req, rewriteResponse.getResource(), diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/StorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/StorageRpc.java index b2eab7eeca42..384f1fd6af11 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/StorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/StorageRpc.java @@ -46,7 +46,8 @@ enum Option { PAGE_TOKEN("pageToken"), DELIMITER("delimiter"), VERSIONS("versions"), - FIELDS("fields"); + FIELDS("fields"), + CUSTOMER_SUPPLIED_KEY("customerSuppliedKey"); private final String value; diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobInfoTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobInfoTest.java index 020e61e9d12e..63ed99c33551 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobInfoTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobInfoTest.java @@ -27,6 +27,7 @@ import com.google.api.services.storage.model.StorageObject; import com.google.cloud.storage.Acl.Project; import com.google.cloud.storage.Acl.User; +import com.google.cloud.storage.BlobInfo.CustomerEncryption; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -61,6 +62,10 @@ public class BlobInfoTest { private static final Long SIZE = 1024L; private static final Long UPDATE_TIME = DELETE_TIME - 1L; private static final Long CREATE_TIME = UPDATE_TIME - 1L; + private static final String ENCRYPTION_ALGORITHM = "AES256"; + private static final String KEY_SHA256 = "keySha"; + private static final CustomerEncryption CUSTOMER_ENCRYPTION = + new CustomerEncryption(ENCRYPTION_ALGORITHM, KEY_SHA256); private static final BlobInfo BLOB_INFO = BlobInfo.builder("b", "n", GENERATION) .acl(ACL) .componentCount(COMPONENT_COUNT) @@ -69,6 +74,7 @@ public class BlobInfoTest { .contentDisposition(CONTENT_DISPOSITION) .contentEncoding(CONTENT_ENCODING) .contentLanguage(CONTENT_LANGUAGE) + .customerEncryption(CUSTOMER_ENCRYPTION) .crc32c(CRC32) .deleteTime(DELETE_TIME) .etag(ETAG) @@ -88,6 +94,12 @@ public class BlobInfoTest { .isDirectory(true) .build(); + @Test + public void testCustomerEncryption() { + assertEquals(ENCRYPTION_ALGORITHM, CUSTOMER_ENCRYPTION.encryptionAlgorithm()); + assertEquals(KEY_SHA256, CUSTOMER_ENCRYPTION.keySha256()); + } + @Test public void testToBuilder() { compareBlobs(BLOB_INFO, BLOB_INFO.toBuilder().build()); @@ -116,6 +128,7 @@ public void testBuilder() { assertEquals(CONTENT_DISPOSITION, BLOB_INFO.contentDisposition()); assertEquals(CONTENT_ENCODING, BLOB_INFO.contentEncoding()); assertEquals(CONTENT_LANGUAGE, BLOB_INFO.contentLanguage()); + assertEquals(CUSTOMER_ENCRYPTION, BLOB_INFO.customerEncryption()); assertEquals(CRC32, BLOB_INFO.crc32c()); assertEquals(DELETE_TIME, BLOB_INFO.deleteTime()); assertEquals(ETAG, BLOB_INFO.etag()); @@ -140,6 +153,7 @@ public void testBuilder() { assertNull(DIRECTORY_INFO.contentDisposition()); assertNull(DIRECTORY_INFO.contentEncoding()); assertNull(DIRECTORY_INFO.contentLanguage()); + assertNull(DIRECTORY_INFO.customerEncryption()); assertNull(DIRECTORY_INFO.crc32c()); assertNull(DIRECTORY_INFO.createTime()); assertNull(DIRECTORY_INFO.deleteTime()); @@ -168,6 +182,7 @@ private void compareBlobs(BlobInfo expected, BlobInfo value) { assertEquals(expected.contentDisposition(), value.contentDisposition()); assertEquals(expected.contentEncoding(), value.contentEncoding()); assertEquals(expected.contentLanguage(), value.contentLanguage()); + assertEquals(expected.customerEncryption(), value.customerEncryption()); assertEquals(expected.crc32c(), value.crc32c()); assertEquals(expected.createTime(), value.createTime()); assertEquals(expected.deleteTime(), value.deleteTime()); @@ -184,8 +199,17 @@ private void compareBlobs(BlobInfo expected, BlobInfo value) { assertEquals(expected.updateTime(), value.updateTime()); } + private void compareCustomerEncryptions(CustomerEncryption expected, CustomerEncryption value) { + assertEquals(expected, value); + assertEquals(expected.encryptionAlgorithm(), value.encryptionAlgorithm()); + assertEquals(expected.keySha256(), value.keySha256()); + assertEquals(expected.hashCode(), value.hashCode()); + } + @Test public void testToPbAndFromPb() { + compareCustomerEncryptions(CUSTOMER_ENCRYPTION, + CustomerEncryption.fromPb(CUSTOMER_ENCRYPTION.toPb())); compareBlobs(BLOB_INFO, BlobInfo.fromPb(BLOB_INFO.toPb())); BlobInfo blobInfo = BlobInfo.builder(BlobId.of("b", "n")).build(); compareBlobs(blobInfo, BlobInfo.fromPb(blobInfo.toPb())); @@ -204,6 +228,7 @@ public void testToPbAndFromPb() { assertNull(blobInfo.contentDisposition()); assertNull(blobInfo.contentEncoding()); assertNull(blobInfo.contentLanguage()); + assertNull(blobInfo.customerEncryption()); assertNull(blobInfo.crc32c()); assertNull(blobInfo.createTime()); assertNull(blobInfo.deleteTime()); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java index 07107f5abb4d..ec73000ce857 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java @@ -35,6 +35,8 @@ import com.google.cloud.storage.Acl.Project.ProjectRole; import com.google.cloud.storage.Acl.Role; import com.google.cloud.storage.Acl.User; +import com.google.cloud.storage.Blob.BlobSourceOption; +import com.google.cloud.storage.Storage.BlobWriteOption; import com.google.cloud.storage.Storage.CopyRequest; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -74,6 +76,10 @@ public class BlobTest { private static final Long SIZE = 1024L; private static final Long UPDATE_TIME = DELETE_TIME - 1L; private static final Long CREATE_TIME = UPDATE_TIME - 1L; + private static final String ENCRYPTION_ALGORITHM = "AES256"; + private static final String KEY_SHA256 = "keySha"; + private static final BlobInfo.CustomerEncryption CUSTOMER_ENCRYPTION = + new BlobInfo.CustomerEncryption(ENCRYPTION_ALGORITHM, KEY_SHA256); private static final BlobInfo FULL_BLOB_INFO = BlobInfo.builder("b", "n", GENERATION) .acl(ACLS) .componentCount(COMPONENT_COUNT) @@ -95,6 +101,7 @@ public class BlobTest { .size(SIZE) .updateTime(UPDATE_TIME) .createTime(CREATE_TIME) + .customerEncryption(CUSTOMER_ENCRYPTION) .build(); private static final BlobInfo BLOB_INFO = BlobInfo.builder("b", "n").metageneration(42L).build(); private static final BlobInfo DIRECTORY_INFO = BlobInfo.builder("b", "n/") @@ -160,6 +167,18 @@ public void testContent() throws Exception { assertArrayEquals(content, blob.content()); } + @Test + public void testContentWithOptions() throws Exception { + initializeExpectedBlob(2); + byte[] content = {1, 2}; + expect(storage.options()).andReturn(mockOptions); + expect(storage.readAllBytes(BLOB_INFO.blobId(), Storage.BlobSourceOption.decryptionKey("key"))) + .andReturn(content); + replay(storage); + initializeBlob(); + assertArrayEquals(content, blob.content(BlobSourceOption.decryptionKey("key"))); + } + @Test public void testReload() throws Exception { initializeExpectedBlob(2); @@ -193,7 +212,7 @@ public void testReloadWithOptions() throws Exception { expect(storage.get(BLOB_INFO.blobId(), options)).andReturn(expectedReloadedBlob); replay(storage); initializeBlob(); - Blob updatedBlob = blob.reload(Blob.BlobSourceOption.metagenerationMatch()); + Blob updatedBlob = blob.reload(BlobSourceOption.metagenerationMatch()); assertEquals(expectedReloadedBlob, updatedBlob); } @@ -290,6 +309,18 @@ public void testReader() throws Exception { assertSame(channel, blob.reader()); } + @Test + public void testReaderWithOptions() throws Exception { + initializeExpectedBlob(2); + ReadChannel channel = createMock(ReadChannel.class); + expect(storage.options()).andReturn(mockOptions); + expect(storage.reader(BLOB_INFO.blobId(), Storage.BlobSourceOption.decryptionKey("key"))) + .andReturn(channel); + replay(storage); + initializeBlob(); + assertSame(channel, blob.reader(BlobSourceOption.decryptionKey("key"))); + } + @Test public void testWriter() throws Exception { initializeExpectedBlob(2); @@ -301,6 +332,18 @@ public void testWriter() throws Exception { assertSame(channel, blob.writer()); } + @Test + public void testWriterWithOptions() throws Exception { + initializeExpectedBlob(2); + BlobWriteChannel channel = createMock(BlobWriteChannel.class); + expect(storage.options()).andReturn(mockOptions); + expect(storage.writer(eq(expectedBlob), eq(BlobWriteOption.encryptionKey("key")))) + .andReturn(channel); + replay(storage); + initializeBlob(); + assertSame(channel, blob.writer(BlobWriteOption.encryptionKey("key"))); + } + @Test public void testSignUrl() throws Exception { initializeExpectedBlob(2); @@ -390,6 +433,8 @@ public void testBuilder() { .contentEncoding(CONTENT_ENCODING) .contentLanguage(CONTENT_LANGUAGE) .crc32c(CRC32) + .createTime(CREATE_TIME) + .customerEncryption(CUSTOMER_ENCRYPTION) .deleteTime(DELETE_TIME) .etag(ETAG) .generatedId(GENERATED_ID) @@ -401,7 +446,6 @@ public void testBuilder() { .selfLink(SELF_LINK) .size(SIZE) .updateTime(UPDATE_TIME) - .createTime(CREATE_TIME) .build(); assertEquals("b", blob.bucket()); assertEquals("n", blob.name()); @@ -414,6 +458,7 @@ public void testBuilder() { assertEquals(CONTENT_LANGUAGE, blob.contentLanguage()); assertEquals(CRC32, blob.crc32c()); assertEquals(CREATE_TIME, blob.createTime()); + assertEquals(CUSTOMER_ENCRYPTION, blob.customerEncryption()); assertEquals(DELETE_TIME, blob.deleteTime()); assertEquals(ETAG, blob.etag()); assertEquals(GENERATED_ID, blob.generatedId()); @@ -442,6 +487,7 @@ public void testBuilder() { assertNull(blob.contentLanguage()); assertNull(blob.crc32c()); assertNull(blob.createTime()); + assertNull(blob.customerEncryption()); assertNull(blob.deleteTime()); assertNull(blob.etag()); assertNull(blob.generatedId()); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java index b7052cadfa9b..e319fb7621bf 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java @@ -319,13 +319,15 @@ public void testCreateWithOptions() throws Exception { expect(storage.options()).andReturn(mockOptions); expect(storage.create(info, content, Storage.BlobTargetOption.generationMatch(), Storage.BlobTargetOption.metagenerationMatch(), - Storage.BlobTargetOption.predefinedAcl(acl))).andReturn(expectedBlob); + Storage.BlobTargetOption.predefinedAcl(acl), + Storage.BlobTargetOption.encryptionKey("key"))).andReturn(expectedBlob); replay(storage); initializeBucket(); Blob blob = bucket.create("n", content, CONTENT_TYPE, Bucket.BlobTargetOption.generationMatch(42L), Bucket.BlobTargetOption.metagenerationMatch(24L), - Bucket.BlobTargetOption.predefinedAcl(acl)); + Bucket.BlobTargetOption.predefinedAcl(acl), + Bucket.BlobTargetOption.encryptionKey("key")); assertEquals(expectedBlob, blob); } @@ -418,14 +420,16 @@ public void testCreateFromStreamWithOptions() throws Exception { expect(storage.options()).andReturn(mockOptions); expect(storage.create(info, streamContent, Storage.BlobWriteOption.generationMatch(), Storage.BlobWriteOption.metagenerationMatch(), Storage.BlobWriteOption.predefinedAcl(acl), - Storage.BlobWriteOption.crc32cMatch(), Storage.BlobWriteOption.md5Match())) + Storage.BlobWriteOption.crc32cMatch(), Storage.BlobWriteOption.md5Match(), + Storage.BlobWriteOption.encryptionKey("key"))) .andReturn(expectedBlob); replay(storage); initializeBucket(); Blob blob = bucket.create("n", streamContent, CONTENT_TYPE, Bucket.BlobWriteOption.generationMatch(42L), Bucket.BlobWriteOption.metagenerationMatch(24L), Bucket.BlobWriteOption.predefinedAcl(acl), - Bucket.BlobWriteOption.crc32cMatch("crc"), Bucket.BlobWriteOption.md5Match("md5")); + Bucket.BlobWriteOption.crc32cMatch("crc"), Bucket.BlobWriteOption.md5Match("md5"), + Bucket.BlobWriteOption.encryptionKey("key")); assertEquals(expectedBlob, blob); } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java index 16a386ccd630..906c6f1cf671 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java @@ -37,6 +37,10 @@ import com.google.cloud.storage.Acl.Project.ProjectRole; import com.google.cloud.storage.Acl.Role; import com.google.cloud.storage.Acl.User; +import com.google.cloud.storage.Storage.BlobSourceOption; +import com.google.cloud.storage.Storage.BlobTargetOption; +import com.google.cloud.storage.Storage.BlobWriteOption; +import com.google.cloud.storage.Storage.BucketSourceOption; import com.google.cloud.storage.Storage.CopyRequest; import com.google.cloud.storage.spi.RpcBatch; import com.google.cloud.storage.spi.StorageRpc; @@ -63,6 +67,7 @@ import java.net.URLDecoder; import java.nio.ByteBuffer; import java.security.InvalidKeyException; +import java.security.Key; import java.security.KeyFactory; import java.security.NoSuchAlgorithmException; import java.security.PrivateKey; @@ -77,6 +82,8 @@ import java.util.Map; import java.util.concurrent.TimeUnit; +import javax.crypto.spec.SecretKeySpec; + public class StorageImplTest { private static final String BUCKET_NAME1 = "b1"; @@ -88,6 +95,9 @@ public class StorageImplTest { private static final String CONTENT_MD5 = "O1R4G1HJSDUISJjoIYmVhQ=="; private static final String CONTENT_CRC32C = "9N3EPQ=="; private static final int DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024; + private static final String BASE64_KEY = "JVzfVl8NLD9FjedFuStegjRfES5ll5zc59CIXw572OA="; + private static final Key KEY = + new SecretKeySpec(BaseEncoding.base64().decode(BASE64_KEY), "AES256"); // BucketInfo objects private static final BucketInfo BUCKET_INFO1 = @@ -113,14 +123,12 @@ public class StorageImplTest { StorageRpc.Option.PREDEFINED_ACL, BUCKET_TARGET_PREDEFINED_ACL.value()); // Blob target options (create, update, compose) - private static final Storage.BlobTargetOption BLOB_TARGET_GENERATION = - Storage.BlobTargetOption.generationMatch(); - private static final Storage.BlobTargetOption BLOB_TARGET_METAGENERATION = - Storage.BlobTargetOption.metagenerationMatch(); - private static final Storage.BlobTargetOption BLOB_TARGET_NOT_EXIST = - Storage.BlobTargetOption.doesNotExist(); - private static final Storage.BlobTargetOption BLOB_TARGET_PREDEFINED_ACL = - Storage.BlobTargetOption.predefinedAcl(Storage.PredefinedAcl.PRIVATE); + private static final BlobTargetOption BLOB_TARGET_GENERATION = BlobTargetOption.generationMatch(); + private static final BlobTargetOption BLOB_TARGET_METAGENERATION = + BlobTargetOption.metagenerationMatch(); + private static final BlobTargetOption BLOB_TARGET_NOT_EXIST = BlobTargetOption.doesNotExist(); + private static final BlobTargetOption BLOB_TARGET_PREDEFINED_ACL = + BlobTargetOption.predefinedAcl(Storage.PredefinedAcl.PRIVATE); private static final Map BLOB_TARGET_OPTIONS_CREATE = ImmutableMap.of( StorageRpc.Option.IF_METAGENERATION_MATCH, BLOB_INFO1.metageneration(), StorageRpc.Option.IF_GENERATION_MATCH, 0L, @@ -133,20 +141,17 @@ public class StorageImplTest { StorageRpc.Option.IF_METAGENERATION_MATCH, BLOB_INFO1.metageneration()); // Blob write options (create, writer) - private static final Storage.BlobWriteOption BLOB_WRITE_METAGENERATION = - Storage.BlobWriteOption.metagenerationMatch(); - private static final Storage.BlobWriteOption BLOB_WRITE_NOT_EXIST = - Storage.BlobWriteOption.doesNotExist(); - private static final Storage.BlobWriteOption BLOB_WRITE_PREDEFINED_ACL = - Storage.BlobWriteOption.predefinedAcl(Storage.PredefinedAcl.PRIVATE); - private static final Storage.BlobWriteOption BLOB_WRITE_MD5_HASH = - Storage.BlobWriteOption.md5Match(); - private static final Storage.BlobWriteOption BLOB_WRITE_CRC2C = - Storage.BlobWriteOption.crc32cMatch(); + private static final BlobWriteOption BLOB_WRITE_METAGENERATION = + BlobWriteOption.metagenerationMatch(); + private static final BlobWriteOption BLOB_WRITE_NOT_EXIST = BlobWriteOption.doesNotExist(); + private static final BlobWriteOption BLOB_WRITE_PREDEFINED_ACL = + BlobWriteOption.predefinedAcl(Storage.PredefinedAcl.PRIVATE); + private static final BlobWriteOption BLOB_WRITE_MD5_HASH = BlobWriteOption.md5Match(); + private static final BlobWriteOption BLOB_WRITE_CRC2C = BlobWriteOption.crc32cMatch(); // Bucket get/source options - private static final Storage.BucketSourceOption BUCKET_SOURCE_METAGENERATION = - Storage.BucketSourceOption.metagenerationMatch(BUCKET_INFO1.metageneration()); + private static final BucketSourceOption BUCKET_SOURCE_METAGENERATION = + BucketSourceOption.metagenerationMatch(BUCKET_INFO1.metageneration()); private static final Map BUCKET_SOURCE_OPTIONS = ImmutableMap.of( StorageRpc.Option.IF_METAGENERATION_MATCH, BUCKET_SOURCE_METAGENERATION.value()); private static final Storage.BucketGetOption BUCKET_GET_METAGENERATION = @@ -172,12 +177,12 @@ public class StorageImplTest { private static final Map BLOB_GET_OPTIONS = ImmutableMap.of( StorageRpc.Option.IF_METAGENERATION_MATCH, BLOB_GET_METAGENERATION.value(), StorageRpc.Option.IF_GENERATION_MATCH, BLOB_GET_GENERATION.value()); - private static final Storage.BlobSourceOption BLOB_SOURCE_METAGENERATION = - Storage.BlobSourceOption.metagenerationMatch(BLOB_INFO1.metageneration()); - private static final Storage.BlobSourceOption BLOB_SOURCE_GENERATION = - Storage.BlobSourceOption.generationMatch(BLOB_INFO1.generation()); - private static final Storage.BlobSourceOption BLOB_SOURCE_GENERATION_FROM_BLOB_ID = - Storage.BlobSourceOption.generationMatch(); + private static final BlobSourceOption BLOB_SOURCE_METAGENERATION = + BlobSourceOption.metagenerationMatch(BLOB_INFO1.metageneration()); + private static final BlobSourceOption BLOB_SOURCE_GENERATION = + BlobSourceOption.generationMatch(BLOB_INFO1.generation()); + private static final BlobSourceOption BLOB_SOURCE_GENERATION_FROM_BLOB_ID = + BlobSourceOption.generationMatch(); private static final Map BLOB_SOURCE_OPTIONS = ImmutableMap.of( StorageRpc.Option.IF_METAGENERATION_MATCH, BLOB_SOURCE_METAGENERATION.value(), StorageRpc.Option.IF_GENERATION_MATCH, BLOB_SOURCE_GENERATION.value()); @@ -218,6 +223,10 @@ public class StorageImplTest { private static final Acl ACL = Acl.of(User.ofAllAuthenticatedUsers(), Role.OWNER); private static final Acl OTHER_ACL = Acl.of(new Project(ProjectRole.OWNERS, "p"), Role.READER); + // Customer supplied encryption key options + private static final Map ENCRYPTION_KEY_OPTIONS = ImmutableMap.of( + StorageRpc.Option.CUSTOMER_SUPPLIED_KEY, BASE64_KEY); + private static final String PRIVATE_KEY_STRING = "MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoG" + "BAL2xolH1zrISQ8+GzOV29BNjjzq4/HIP8Psd1+cZb81vDklSF+95wB250MSE0BDc81pvIMwj5OmIfLg1NY6uB" + "1xavOPpVdx1z664AGc/BEJ1zInXGXaQ6s+SxGenVq40Yws57gikQGMZjttpf1Qbz4DjkxsbRoeaRHn06n9pH1e" @@ -913,6 +922,26 @@ public void testCopyWithOptions() { assertTrue(!writer.isDone()); } + @Test + public void testCopyWithEncryptionKey() { + CopyRequest request = Storage.CopyRequest.builder() + .source(BLOB_INFO2.blobId()) + .sourceOptions(BlobSourceOption.decryptionKey(KEY)) + .target(BLOB_INFO1, BlobTargetOption.encryptionKey(BASE64_KEY)) + .build(); + StorageRpc.RewriteRequest rpcRequest = new StorageRpc.RewriteRequest(request.source().toPb(), + ENCRYPTION_KEY_OPTIONS, true, request.target().toPb(), ENCRYPTION_KEY_OPTIONS, null); + StorageRpc.RewriteResponse rpcResponse = new StorageRpc.RewriteResponse(rpcRequest, null, 42L, + false, "token", 21L); + EasyMock.expect(storageRpcMock.openRewrite(rpcRequest)).andReturn(rpcResponse); + EasyMock.replay(storageRpcMock); + initializeService(); + CopyWriter writer = storage.copy(request); + assertEquals(42L, writer.blobSize()); + assertEquals(21L, writer.totalBytesCopied()); + assertTrue(!writer.isDone()); + } + @Test public void testCopyWithOptionsFromBlobId() { CopyRequest request = Storage.CopyRequest.builder() @@ -979,6 +1008,30 @@ public void testReadAllBytesWithOptions() { assertArrayEquals(BLOB_CONTENT, readBytes); } + @Test + public void testReadAllBytesWithDecriptionKey() { + EasyMock.expect( + storageRpcMock.load(BlobId.of(BUCKET_NAME1, BLOB_NAME1).toPb(), ENCRYPTION_KEY_OPTIONS)) + .andReturn(BLOB_CONTENT); + EasyMock.replay(storageRpcMock); + initializeService(); + byte[] readBytes = storage.readAllBytes(BUCKET_NAME1, BLOB_NAME1, + BlobSourceOption.decryptionKey(KEY)); + assertArrayEquals(BLOB_CONTENT, readBytes); + } + + @Test + public void testReadAllBytesWithBase64DecriptionKey() { + EasyMock.expect( + storageRpcMock.load(BlobId.of(BUCKET_NAME1, BLOB_NAME1).toPb(), ENCRYPTION_KEY_OPTIONS)) + .andReturn(BLOB_CONTENT); + EasyMock.replay(storageRpcMock); + initializeService(); + byte[] readBytes = storage.readAllBytes(BUCKET_NAME1, BLOB_NAME1, + BlobSourceOption.decryptionKey(BASE64_KEY)); + assertArrayEquals(BLOB_CONTENT, readBytes); + } + @Test public void testReadAllBytesWithOptionsFromBlobId() { EasyMock.expect( @@ -1028,6 +1081,36 @@ public void testReaderWithOptions() throws IOException { channel.read(ByteBuffer.allocate(42)); } + @Test + public void testReaderWithDecryptionKey() throws IOException { + byte[] result = new byte[DEFAULT_CHUNK_SIZE]; + EasyMock.expect( + storageRpcMock.read(BLOB_INFO2.toPb(), ENCRYPTION_KEY_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) + .andReturn(StorageRpc.Tuple.of("etag", result)); + EasyMock.replay(storageRpcMock); + initializeService(); + ReadChannel channel = + storage.reader(BUCKET_NAME1, BLOB_NAME2, BlobSourceOption.decryptionKey(KEY)); + assertNotNull(channel); + assertTrue(channel.isOpen()); + channel.read(ByteBuffer.allocate(42)); + } + + @Test + public void testReaderWithBase64DecryptionKey() throws IOException { + byte[] result = new byte[DEFAULT_CHUNK_SIZE]; + EasyMock.expect( + storageRpcMock.read(BLOB_INFO2.toPb(), ENCRYPTION_KEY_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) + .andReturn(StorageRpc.Tuple.of("etag", result)); + EasyMock.replay(storageRpcMock); + initializeService(); + ReadChannel channel = + storage.reader(BUCKET_NAME1, BLOB_NAME2, BlobSourceOption.decryptionKey(BASE64_KEY)); + assertNotNull(channel); + assertTrue(channel.isOpen()); + channel.read(ByteBuffer.allocate(42)); + } + @Test public void testReaderWithOptionsFromBlobId() throws IOException { byte[] result = new byte[DEFAULT_CHUNK_SIZE]; @@ -1070,6 +1153,30 @@ public void testWriterWithOptions() { assertTrue(channel.isOpen()); } + @Test + public void testWriterWithEncryptionKey() { + BlobInfo info = BLOB_INFO1.toBuilder().md5(null).crc32c(null).build(); + EasyMock.expect(storageRpcMock.open(info.toPb(), ENCRYPTION_KEY_OPTIONS)) + .andReturn("upload-id"); + EasyMock.replay(storageRpcMock); + initializeService(); + WriteChannel channel = storage.writer(info, BlobWriteOption.encryptionKey(KEY)); + assertNotNull(channel); + assertTrue(channel.isOpen()); + } + + @Test + public void testWriterWithBase64EncryptionKey() { + BlobInfo info = BLOB_INFO1.toBuilder().md5(null).crc32c(null).build(); + EasyMock.expect(storageRpcMock.open(info.toPb(), ENCRYPTION_KEY_OPTIONS)) + .andReturn("upload-id"); + EasyMock.replay(storageRpcMock); + initializeService(); + WriteChannel channel = storage.writer(info, BlobWriteOption.encryptionKey(BASE64_KEY)); + assertNotNull(channel); + assertTrue(channel.isOpen()); + } + @Test public void testSignUrl() throws NoSuchAlgorithmException, InvalidKeyException, SignatureException, UnsupportedEncodingException { diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index e91486944bd7..e45fb117bfe7 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -52,6 +52,7 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import com.google.common.io.BaseEncoding; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -64,6 +65,9 @@ import java.net.URL; import java.net.URLConnection; import java.nio.ByteBuffer; +import java.security.Key; +import java.security.NoSuchAlgorithmException; +import java.security.spec.InvalidKeySpecException; import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; @@ -76,6 +80,8 @@ import java.util.logging.Level; import java.util.logging.Logger; +import javax.crypto.spec.SecretKeySpec; + public class ITStorageTest { private static Storage storage; @@ -86,9 +92,13 @@ public class ITStorageTest { private static final byte[] BLOB_BYTE_CONTENT = {0xD, 0xE, 0xA, 0xD}; private static final String BLOB_STRING_CONTENT = "Hello Google Cloud Storage!"; private static final int MAX_BATCH_SIZE = 100; + private static final String BASE64_KEY = "JVzfVl8NLD9FjedFuStegjRfES5ll5zc59CIXw572OA="; + private static final String OTHER_BASE64_KEY = "IcOIQGlliNr5pr3vJb63l+XMqc7NjXqjfw/deBoNxPA="; + private static final Key KEY = + new SecretKeySpec(BaseEncoding.base64().decode(BASE64_KEY), "AES256"); @BeforeClass - public static void beforeClass() { + public static void beforeClass() throws NoSuchAlgorithmException, InvalidKeySpecException { RemoteStorageHelper helper = RemoteStorageHelper.create(); storage = helper.options().service(); storage.create(BucketInfo.of(BUCKET)); @@ -159,6 +169,21 @@ public void testCreateBlob() { assertTrue(remoteBlob.delete()); } + @Test + public void testCreateBlobWithEncryptionKey() { + String blobName = "test-create-with-customer-key-blob"; + BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build(); + Blob remoteBlob = + storage.create(blob, BLOB_BYTE_CONTENT, Storage.BlobTargetOption.encryptionKey(KEY)); + assertNotNull(remoteBlob); + assertEquals(blob.bucket(), remoteBlob.bucket()); + assertEquals(blob.name(), remoteBlob.name()); + byte[] readBytes = + storage.readAllBytes(BUCKET, blobName, Storage.BlobSourceOption.decryptionKey(BASE64_KEY)); + assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); + assertTrue(remoteBlob.delete()); + } + @Test public void testCreateEmptyBlob() { String blobName = "test-create-empty-blob"; @@ -694,6 +719,48 @@ public void testCopyBlob() { assertTrue(storage.delete(BUCKET, targetBlobName)); } + @Test + public void testCopyBlobWithEncryptionKeys() { + String sourceBlobName = "test-copy-blob-encryption-key-source"; + BlobId source = BlobId.of(BUCKET, sourceBlobName); + ImmutableMap metadata = ImmutableMap.of("k", "v"); + Blob remoteBlob = storage.create(BlobInfo.builder(source).build(), BLOB_BYTE_CONTENT, + Storage.BlobTargetOption.encryptionKey(KEY)); + assertNotNull(remoteBlob); + String targetBlobName = "test-copy-blob-encryption-key-target"; + BlobInfo target = BlobInfo.builder(BUCKET, targetBlobName) + .contentType(CONTENT_TYPE) + .metadata(metadata) + .build(); + Storage.CopyRequest req = Storage.CopyRequest.builder() + .source(source) + .target(target, Storage.BlobTargetOption.encryptionKey(OTHER_BASE64_KEY)) + .sourceOptions(Storage.BlobSourceOption.decryptionKey(BASE64_KEY)) + .build(); + CopyWriter copyWriter = storage.copy(req); + assertEquals(BUCKET, copyWriter.result().bucket()); + assertEquals(targetBlobName, copyWriter.result().name()); + assertEquals(CONTENT_TYPE, copyWriter.result().contentType()); + assertArrayEquals(BLOB_BYTE_CONTENT, + copyWriter.result().content(Blob.BlobSourceOption.decryptionKey(OTHER_BASE64_KEY))); + assertEquals(metadata, copyWriter.result().metadata()); + assertTrue(copyWriter.isDone()); + req = Storage.CopyRequest.builder() + .source(source) + .target(target) + .sourceOptions(Storage.BlobSourceOption.decryptionKey(BASE64_KEY)) + .build(); + copyWriter = storage.copy(req); + assertEquals(BUCKET, copyWriter.result().bucket()); + assertEquals(targetBlobName, copyWriter.result().name()); + assertEquals(CONTENT_TYPE, copyWriter.result().contentType()); + assertArrayEquals(BLOB_BYTE_CONTENT, copyWriter.result().content()); + assertEquals(metadata, copyWriter.result().metadata()); + assertTrue(copyWriter.isDone()); + assertTrue(remoteBlob.delete()); + assertTrue(storage.delete(BUCKET, targetBlobName)); + } + @Test public void testCopyBlobUpdateMetadata() { String sourceBlobName = "test-copy-blob-update-metadata-source"; @@ -944,6 +1011,31 @@ public void testReadAndWriteChannels() throws IOException { assertTrue(storage.delete(BUCKET, blobName)); } + @Test + public void testReadAndWriteChannelWithEncryptionKey() throws IOException { + String blobName = "test-read-write-channel-with-customer-key-blob"; + BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build(); + byte[] stringBytes; + try (WriteChannel writer = storage.writer(blob, + Storage.BlobWriteOption.encryptionKey(BASE64_KEY))) { + stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8); + writer.write(ByteBuffer.wrap(BLOB_BYTE_CONTENT)); + writer.write(ByteBuffer.wrap(stringBytes)); + } + ByteBuffer readBytes; + ByteBuffer readStringBytes; + try (ReadChannel reader = + storage.reader(blob.blobId(), Storage.BlobSourceOption.decryptionKey(KEY))) { + readBytes = ByteBuffer.allocate(BLOB_BYTE_CONTENT.length); + readStringBytes = ByteBuffer.allocate(stringBytes.length); + reader.read(readBytes); + reader.read(readStringBytes); + } + assertArrayEquals(BLOB_BYTE_CONTENT, readBytes.array()); + assertEquals(BLOB_STRING_CONTENT, new String(readStringBytes.array(), UTF_8)); + assertTrue(storage.delete(BUCKET, blobName)); + } + @Test public void testReadAndWriteChannelsWithDifferentFileSize() throws IOException { String blobNamePrefix = "test-read-and-write-channels-blob-"; From b0b6aa09415bb927554c9a7ea6c42b3d22ea39ef Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Mon, 12 Sep 2016 16:02:06 +0200 Subject: [PATCH 2/2] Better test coverage for user-supplied encryption keys --- .../com/google/cloud/storage/BlobTest.java | 35 ++++-- .../com/google/cloud/storage/BucketTest.java | 48 +++++++- .../google/cloud/storage/StorageImplTest.java | 110 ++++++++++++------ 3 files changed, 142 insertions(+), 51 deletions(-) diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java index ec73000ce857..577e74685f84 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java @@ -40,6 +40,7 @@ import com.google.cloud.storage.Storage.CopyRequest; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.io.BaseEncoding; import org.easymock.Capture; import org.junit.After; @@ -47,10 +48,13 @@ import org.junit.Test; import java.net.URL; +import java.security.Key; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import javax.crypto.spec.SecretKeySpec; + public class BlobTest { private static final Acl ACL = Acl.of(User.ofAllAuthenticatedUsers(), Role.OWNER); @@ -108,6 +112,9 @@ public class BlobTest { .size(0L) .isDirectory(true) .build(); + private static final String BASE64_KEY = "JVzfVl8NLD9FjedFuStegjRfES5ll5zc59CIXw572OA="; + private static final Key KEY = + new SecretKeySpec(BaseEncoding.base64().decode(BASE64_KEY), "AES256"); private Storage storage; private Blob blob; @@ -168,15 +175,17 @@ public void testContent() throws Exception { } @Test - public void testContentWithOptions() throws Exception { + public void testContentWithDecryptionKey() throws Exception { initializeExpectedBlob(2); byte[] content = {1, 2}; expect(storage.options()).andReturn(mockOptions); - expect(storage.readAllBytes(BLOB_INFO.blobId(), Storage.BlobSourceOption.decryptionKey("key"))) - .andReturn(content); + expect(storage.readAllBytes(BLOB_INFO.blobId(), + Storage.BlobSourceOption.decryptionKey(BASE64_KEY))) + .andReturn(content).times(2); replay(storage); initializeBlob(); - assertArrayEquals(content, blob.content(BlobSourceOption.decryptionKey("key"))); + assertArrayEquals(content, blob.content(BlobSourceOption.decryptionKey(BASE64_KEY))); + assertArrayEquals(content, blob.content(BlobSourceOption.decryptionKey(KEY))); } @Test @@ -310,15 +319,16 @@ public void testReader() throws Exception { } @Test - public void testReaderWithOptions() throws Exception { + public void testReaderWithDecryptionKey() throws Exception { initializeExpectedBlob(2); ReadChannel channel = createMock(ReadChannel.class); expect(storage.options()).andReturn(mockOptions); - expect(storage.reader(BLOB_INFO.blobId(), Storage.BlobSourceOption.decryptionKey("key"))) - .andReturn(channel); + expect(storage.reader(BLOB_INFO.blobId(), Storage.BlobSourceOption.decryptionKey(BASE64_KEY))) + .andReturn(channel).times(2); replay(storage); initializeBlob(); - assertSame(channel, blob.reader(BlobSourceOption.decryptionKey("key"))); + assertSame(channel, blob.reader(BlobSourceOption.decryptionKey(BASE64_KEY))); + assertSame(channel, blob.reader(BlobSourceOption.decryptionKey(KEY))); } @Test @@ -333,15 +343,16 @@ public void testWriter() throws Exception { } @Test - public void testWriterWithOptions() throws Exception { + public void testWriterWithEncryptionKey() throws Exception { initializeExpectedBlob(2); BlobWriteChannel channel = createMock(BlobWriteChannel.class); expect(storage.options()).andReturn(mockOptions); - expect(storage.writer(eq(expectedBlob), eq(BlobWriteOption.encryptionKey("key")))) - .andReturn(channel); + expect(storage.writer(eq(expectedBlob), eq(BlobWriteOption.encryptionKey(BASE64_KEY)))) + .andReturn(channel).times(2); replay(storage); initializeBlob(); - assertSame(channel, blob.writer(BlobWriteOption.encryptionKey("key"))); + assertSame(channel, blob.writer(BlobWriteOption.encryptionKey(BASE64_KEY))); + assertSame(channel, blob.writer(BlobWriteOption.encryptionKey(KEY))); } @Test diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java index e319fb7621bf..ce1aa90bf869 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java @@ -38,6 +38,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import com.google.common.io.BaseEncoding; import org.junit.After; import org.junit.Before; @@ -47,10 +48,13 @@ import java.io.ByteArrayInputStream; import java.io.InputStream; +import java.security.Key; import java.util.Collections; import java.util.Iterator; import java.util.List; +import javax.crypto.spec.SecretKeySpec; + public class BucketTest { private static final Acl ACL = Acl.of(User.ofAllAuthenticatedUsers(), Role.OWNER); @@ -91,6 +95,9 @@ public class BucketTest { .build(); private static final BucketInfo BUCKET_INFO = BucketInfo.builder("b").metageneration(42L).build(); private static final String CONTENT_TYPE = "text/plain"; + private static final String BASE64_KEY = "JVzfVl8NLD9FjedFuStegjRfES5ll5zc59CIXw572OA="; + private static final Key KEY = + new SecretKeySpec(BaseEncoding.base64().decode(BASE64_KEY), "AES256"); private Storage storage; private Storage serviceMockReturnsOptions = createMock(Storage.class); @@ -320,14 +327,30 @@ public void testCreateWithOptions() throws Exception { expect(storage.create(info, content, Storage.BlobTargetOption.generationMatch(), Storage.BlobTargetOption.metagenerationMatch(), Storage.BlobTargetOption.predefinedAcl(acl), - Storage.BlobTargetOption.encryptionKey("key"))).andReturn(expectedBlob); + Storage.BlobTargetOption.encryptionKey(BASE64_KEY))).andReturn(expectedBlob); replay(storage); initializeBucket(); Blob blob = bucket.create("n", content, CONTENT_TYPE, Bucket.BlobTargetOption.generationMatch(42L), Bucket.BlobTargetOption.metagenerationMatch(24L), Bucket.BlobTargetOption.predefinedAcl(acl), - Bucket.BlobTargetOption.encryptionKey("key")); + Bucket.BlobTargetOption.encryptionKey(BASE64_KEY)); + assertEquals(expectedBlob, blob); + } + + @Test + public void testCreateWithEncryptionKey() throws Exception { + initializeExpectedBucket(5); + BlobInfo info = BlobInfo.builder(BlobId.of("b", "n")).contentType(CONTENT_TYPE).build(); + Blob expectedBlob = new Blob(serviceMockReturnsOptions, new BlobInfo.BuilderImpl(info)); + byte[] content = {0xD, 0xE, 0xA, 0xD}; + expect(storage.options()).andReturn(mockOptions); + expect(storage.create(info, content, Storage.BlobTargetOption.encryptionKey(KEY))) + .andReturn(expectedBlob); + replay(storage); + initializeBucket(); + Blob blob = + bucket.create("n", content, CONTENT_TYPE, Bucket.BlobTargetOption.encryptionKey(KEY)); assertEquals(expectedBlob, blob); } @@ -421,7 +444,7 @@ public void testCreateFromStreamWithOptions() throws Exception { expect(storage.create(info, streamContent, Storage.BlobWriteOption.generationMatch(), Storage.BlobWriteOption.metagenerationMatch(), Storage.BlobWriteOption.predefinedAcl(acl), Storage.BlobWriteOption.crc32cMatch(), Storage.BlobWriteOption.md5Match(), - Storage.BlobWriteOption.encryptionKey("key"))) + Storage.BlobWriteOption.encryptionKey(BASE64_KEY))) .andReturn(expectedBlob); replay(storage); initializeBucket(); @@ -429,7 +452,24 @@ public void testCreateFromStreamWithOptions() throws Exception { Bucket.BlobWriteOption.generationMatch(42L), Bucket.BlobWriteOption.metagenerationMatch(24L), Bucket.BlobWriteOption.predefinedAcl(acl), Bucket.BlobWriteOption.crc32cMatch("crc"), Bucket.BlobWriteOption.md5Match("md5"), - Bucket.BlobWriteOption.encryptionKey("key")); + Bucket.BlobWriteOption.encryptionKey(BASE64_KEY)); + assertEquals(expectedBlob, blob); + } + + @Test + public void testCreateFromStreamWithEncryptionKey() throws Exception { + initializeExpectedBucket(5); + BlobInfo info = BlobInfo.builder(BlobId.of("b", "n")).contentType(CONTENT_TYPE).build(); + Blob expectedBlob = new Blob(serviceMockReturnsOptions, new BlobInfo.BuilderImpl(info)); + byte[] content = {0xD, 0xE, 0xA, 0xD}; + InputStream streamContent = new ByteArrayInputStream(content); + expect(storage.options()).andReturn(mockOptions); + expect(storage.create(info, streamContent, Storage.BlobWriteOption.encryptionKey(KEY))) + .andReturn(expectedBlob); + replay(storage); + initializeBucket(); + Blob blob = + bucket.create("n", streamContent, CONTENT_TYPE, Bucket.BlobWriteOption.encryptionKey(KEY)); assertEquals(expectedBlob, blob); } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java index 906c6f1cf671..3296bf58154f 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java @@ -402,6 +402,33 @@ public void testCreateBlobWithOptions() throws IOException { assertEquals(-1, byteStream.read(streamBytes)); } + @Test + public void testCreateBlobWithEncryptionKey() throws IOException { + Capture capturedStream = Capture.newInstance(); + EasyMock.expect(storageRpcMock.create( + EasyMock.eq(BLOB_INFO1.toBuilder().md5(CONTENT_MD5).crc32c(CONTENT_CRC32C).build().toPb()), + EasyMock.capture(capturedStream), + EasyMock.eq(ENCRYPTION_KEY_OPTIONS))) + .andReturn(BLOB_INFO1.toPb()) + .times(2); + EasyMock.replay(storageRpcMock); + initializeService(); + Blob blob = storage.create(BLOB_INFO1, BLOB_CONTENT, BlobTargetOption.encryptionKey(KEY)); + assertEquals(expectedBlob1, blob); + ByteArrayInputStream byteStream = capturedStream.getValue(); + byte[] streamBytes = new byte[BLOB_CONTENT.length]; + assertEquals(BLOB_CONTENT.length, byteStream.read(streamBytes)); + assertArrayEquals(BLOB_CONTENT, streamBytes); + assertEquals(-1, byteStream.read(streamBytes)); + blob = storage.create(BLOB_INFO1, BLOB_CONTENT, BlobTargetOption.encryptionKey(BASE64_KEY)); + assertEquals(expectedBlob1, blob); + byteStream = capturedStream.getValue(); + streamBytes = new byte[BLOB_CONTENT.length]; + assertEquals(BLOB_CONTENT.length, byteStream.read(streamBytes)); + assertArrayEquals(BLOB_CONTENT, streamBytes); + assertEquals(-1, byteStream.read(streamBytes)); + } + @Test public void testCreateBlobFromStream() { ByteArrayInputStream fileStream = new ByteArrayInputStream(BLOB_CONTENT); @@ -416,6 +443,24 @@ public void testCreateBlobFromStream() { assertEquals(expectedBlob1, blob); } + @Test + public void testCreateBlobFromStreamWithEncryptionKey() throws IOException { + ByteArrayInputStream fileStream = new ByteArrayInputStream(BLOB_CONTENT); + BlobInfo.Builder infoBuilder = BLOB_INFO1.toBuilder(); + BlobInfo infoWithHashes = infoBuilder.md5(CONTENT_MD5).crc32c(CONTENT_CRC32C).build(); + BlobInfo infoWithoutHashes = infoBuilder.md5(null).crc32c(null).build(); + EasyMock.expect( + storageRpcMock.create(infoWithoutHashes.toPb(), fileStream, ENCRYPTION_KEY_OPTIONS)) + .andReturn(BLOB_INFO1.toPb()).times(2); + EasyMock.replay(storageRpcMock); + initializeService(); + Blob blob = + storage.create(infoWithHashes, fileStream, BlobWriteOption.encryptionKey(BASE64_KEY)); + assertEquals(expectedBlob1, blob); + blob = storage.create(infoWithHashes, fileStream, BlobWriteOption.encryptionKey(BASE64_KEY)); + assertEquals(expectedBlob1, blob); + } + @Test public void testGetBucket() { EasyMock.expect(storageRpcMock.get(BucketInfo.of(BUCKET_NAME1).toPb(), EMPTY_RPC_OPTIONS)) @@ -933,13 +978,22 @@ public void testCopyWithEncryptionKey() { ENCRYPTION_KEY_OPTIONS, true, request.target().toPb(), ENCRYPTION_KEY_OPTIONS, null); StorageRpc.RewriteResponse rpcResponse = new StorageRpc.RewriteResponse(rpcRequest, null, 42L, false, "token", 21L); - EasyMock.expect(storageRpcMock.openRewrite(rpcRequest)).andReturn(rpcResponse); + EasyMock.expect(storageRpcMock.openRewrite(rpcRequest)).andReturn(rpcResponse).times(2); EasyMock.replay(storageRpcMock); initializeService(); CopyWriter writer = storage.copy(request); assertEquals(42L, writer.blobSize()); assertEquals(21L, writer.totalBytesCopied()); assertTrue(!writer.isDone()); + request = Storage.CopyRequest.builder() + .source(BLOB_INFO2.blobId()) + .sourceOptions(BlobSourceOption.decryptionKey(BASE64_KEY)) + .target(BLOB_INFO1, BlobTargetOption.encryptionKey(KEY)) + .build(); + writer = storage.copy(request); + assertEquals(42L, writer.blobSize()); + assertEquals(21L, writer.totalBytesCopied()); + assertTrue(!writer.isDone()); } @Test @@ -1012,35 +1066,41 @@ public void testReadAllBytesWithOptions() { public void testReadAllBytesWithDecriptionKey() { EasyMock.expect( storageRpcMock.load(BlobId.of(BUCKET_NAME1, BLOB_NAME1).toPb(), ENCRYPTION_KEY_OPTIONS)) - .andReturn(BLOB_CONTENT); + .andReturn(BLOB_CONTENT).times(2); EasyMock.replay(storageRpcMock); initializeService(); byte[] readBytes = storage.readAllBytes(BUCKET_NAME1, BLOB_NAME1, BlobSourceOption.decryptionKey(KEY)); assertArrayEquals(BLOB_CONTENT, readBytes); + readBytes = storage.readAllBytes(BUCKET_NAME1, BLOB_NAME1, + BlobSourceOption.decryptionKey(BASE64_KEY)); + assertArrayEquals(BLOB_CONTENT, readBytes); } @Test - public void testReadAllBytesWithBase64DecriptionKey() { + public void testReadAllBytesFromBlobIdWithOptions() { EasyMock.expect( - storageRpcMock.load(BlobId.of(BUCKET_NAME1, BLOB_NAME1).toPb(), ENCRYPTION_KEY_OPTIONS)) + storageRpcMock.load(BLOB_INFO1.blobId().toPb(), BLOB_SOURCE_OPTIONS)) .andReturn(BLOB_CONTENT); EasyMock.replay(storageRpcMock); initializeService(); - byte[] readBytes = storage.readAllBytes(BUCKET_NAME1, BLOB_NAME1, - BlobSourceOption.decryptionKey(BASE64_KEY)); + byte[] readBytes = storage.readAllBytes(BLOB_INFO1.blobId(), + BLOB_SOURCE_GENERATION_FROM_BLOB_ID, BLOB_SOURCE_METAGENERATION); assertArrayEquals(BLOB_CONTENT, readBytes); } @Test - public void testReadAllBytesWithOptionsFromBlobId() { + public void testReadAllBytesFromBlobIdWithDecriptionKey() { EasyMock.expect( - storageRpcMock.load(BLOB_INFO1.blobId().toPb(), BLOB_SOURCE_OPTIONS)) - .andReturn(BLOB_CONTENT); + storageRpcMock.load(BLOB_INFO1.blobId().toPb(), ENCRYPTION_KEY_OPTIONS)) + .andReturn(BLOB_CONTENT).times(2); EasyMock.replay(storageRpcMock); initializeService(); - byte[] readBytes = storage.readAllBytes(BLOB_INFO1.blobId(), - BLOB_SOURCE_GENERATION_FROM_BLOB_ID, BLOB_SOURCE_METAGENERATION); + byte[] readBytes = + storage.readAllBytes(BLOB_INFO1.blobId(), BlobSourceOption.decryptionKey(KEY)); + assertArrayEquals(BLOB_CONTENT, readBytes); + readBytes = + storage.readAllBytes(BLOB_INFO1.blobId(), BlobSourceOption.decryptionKey(BASE64_KEY)); assertArrayEquals(BLOB_CONTENT, readBytes); } @@ -1086,7 +1146,7 @@ public void testReaderWithDecryptionKey() throws IOException { byte[] result = new byte[DEFAULT_CHUNK_SIZE]; EasyMock.expect( storageRpcMock.read(BLOB_INFO2.toPb(), ENCRYPTION_KEY_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) - .andReturn(StorageRpc.Tuple.of("etag", result)); + .andReturn(StorageRpc.Tuple.of("etag", result)).times(2); EasyMock.replay(storageRpcMock); initializeService(); ReadChannel channel = @@ -1094,18 +1154,7 @@ public void testReaderWithDecryptionKey() throws IOException { assertNotNull(channel); assertTrue(channel.isOpen()); channel.read(ByteBuffer.allocate(42)); - } - - @Test - public void testReaderWithBase64DecryptionKey() throws IOException { - byte[] result = new byte[DEFAULT_CHUNK_SIZE]; - EasyMock.expect( - storageRpcMock.read(BLOB_INFO2.toPb(), ENCRYPTION_KEY_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) - .andReturn(StorageRpc.Tuple.of("etag", result)); - EasyMock.replay(storageRpcMock); - initializeService(); - ReadChannel channel = - storage.reader(BUCKET_NAME1, BLOB_NAME2, BlobSourceOption.decryptionKey(BASE64_KEY)); + channel = storage.reader(BUCKET_NAME1, BLOB_NAME2, BlobSourceOption.decryptionKey(BASE64_KEY)); assertNotNull(channel); assertTrue(channel.isOpen()); channel.read(ByteBuffer.allocate(42)); @@ -1157,22 +1206,13 @@ public void testWriterWithOptions() { public void testWriterWithEncryptionKey() { BlobInfo info = BLOB_INFO1.toBuilder().md5(null).crc32c(null).build(); EasyMock.expect(storageRpcMock.open(info.toPb(), ENCRYPTION_KEY_OPTIONS)) - .andReturn("upload-id"); + .andReturn("upload-id").times(2); EasyMock.replay(storageRpcMock); initializeService(); WriteChannel channel = storage.writer(info, BlobWriteOption.encryptionKey(KEY)); assertNotNull(channel); assertTrue(channel.isOpen()); - } - - @Test - public void testWriterWithBase64EncryptionKey() { - BlobInfo info = BLOB_INFO1.toBuilder().md5(null).crc32c(null).build(); - EasyMock.expect(storageRpcMock.open(info.toPb(), ENCRYPTION_KEY_OPTIONS)) - .andReturn("upload-id"); - EasyMock.replay(storageRpcMock); - initializeService(); - WriteChannel channel = storage.writer(info, BlobWriteOption.encryptionKey(BASE64_KEY)); + channel = storage.writer(info, BlobWriteOption.encryptionKey(BASE64_KEY)); assertNotNull(channel); assertTrue(channel.isOpen()); }