From 031b1027b3c1bf974f4b94b734202a6c5f27193e Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Thu, 26 Apr 2018 10:07:28 -0700 Subject: [PATCH 01/13] Adding base GCS CMEK implementation from rossjudson@ --- .../java/com/google/cloud/storage/Blob.java | 6 ++++ .../com/google/cloud/storage/BlobInfo.java | 25 +++++++++++++++ .../java/com/google/cloud/storage/Bucket.java | 6 ++++ .../com/google/cloud/storage/BucketInfo.java | 31 ++++++++++++++++++- 4 files changed, 67 insertions(+), 1 deletion(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java index 887ab004944d..14959a2c58ca 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java @@ -403,6 +403,12 @@ Builder setCustomerEncryption(CustomerEncryption customerEncryption) { return this; } + @Override + Builder setKmsKey(String kmsKey) { + infoBuilder.setKmsKey(kmsKey); + return this; + } + @Override public Blob build() { return new Blob(storage, infoBuilder); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java index a903500455de..b5f6674046e8 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java @@ -83,6 +83,7 @@ public StorageObject apply(BlobInfo blobInfo) { private final Integer componentCount; private final boolean isDirectory; private final CustomerEncryption customerEncryption; + private final String kmsKey; /** * This class is meant for internal use only. Users are discouraged from using this class. @@ -266,6 +267,8 @@ public abstract static class Builder { abstract Builder setCustomerEncryption(CustomerEncryption customerEncryption); + abstract Builder setKmsKey(String kmsKey); + /** * Creates a {@code BlobInfo} object. */ @@ -298,6 +301,7 @@ static final class BuilderImpl extends Builder { private Boolean isDirectory; private CustomerEncryption customerEncryption; private StorageClass storageClass; + private String kmsKey; BuilderImpl(BlobId blobId) { this.blobId = blobId; @@ -328,6 +332,7 @@ static final class BuilderImpl extends Builder { createTime = blobInfo.createTime; isDirectory = blobInfo.isDirectory; storageClass = blobInfo.storageClass; + kmsKey = blobInfo.kmsKey; } @Override @@ -475,6 +480,12 @@ Builder setCustomerEncryption(CustomerEncryption customerEncryption) { return this; } + @Override + Builder setKmsKey(String kmsKey) { + this.kmsKey = kmsKey; + return this; + } + @Override public BlobInfo build() { checkNotNull(blobId); @@ -507,6 +518,7 @@ public BlobInfo build() { createTime = builder.createTime; isDirectory = firstNonNull(builder.isDirectory, Boolean.FALSE); storageClass = builder.storageClass; + kmsKey = builder.kmsKey; } /** @@ -737,6 +749,13 @@ public StorageClass getStorageClass() { return storageClass; } + /** + * Returns the Cloud KMS key used to encrypt the blob, if any. + */ + public String getKmsKey() { + return kmsKey; + } + /** * Returns a builder for the current blob. */ @@ -809,6 +828,9 @@ public ObjectAccessControl apply(Acl acl) { if (customerEncryption != null) { storageObject.setCustomerEncryption(customerEncryption.toPb()); } + if (kmsKey != null) { + storageObject.setKmsKeyName(kmsKey); + } storageObject.setMetadata(pbMetadata); storageObject.setCacheControl(cacheControl); storageObject.setContentEncoding(contentEncoding); @@ -939,6 +961,9 @@ public Acl apply(ObjectAccessControl objectAccessControl) { if (storageObject.getStorageClass() != null) { builder.setStorageClass(StorageClass.valueOf(storageObject.getStorageClass())); } + if (storageObject.getKmsKeyName() != null) { + builder.setKmsKey(storageObject.getKmsKeyName()); + } return builder.build(); } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java index b4855e96a411..9b07f4567f1a 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java @@ -615,6 +615,12 @@ public Builder setLabels(Map labels) { return this; } + @Override + public Builder setDefaultKmsKeyName(String defaultKmsKeyName) { + infoBuilder.setDefaultKmsKeyName(defaultKmsKeyName); + return this; + } + @Override public Bucket build() { return new Bucket(storage, infoBuilder); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java index 82a98a75ad82..bfdce530aac0 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java @@ -25,6 +25,7 @@ import com.google.api.client.util.DateTime; import com.google.api.services.storage.model.*; import com.google.api.services.storage.model.Bucket; +import com.google.api.services.storage.model.Bucket.Encryption; import com.google.api.services.storage.model.Bucket.Lifecycle; import com.google.api.services.storage.model.Bucket.Lifecycle.Rule; import com.google.api.services.storage.model.Bucket.Owner; @@ -85,6 +86,7 @@ public com.google.api.services.storage.model.Bucket apply(BucketInfo bucketInfo) private final String location; private final StorageClass storageClass; private final Map labels; + private final String defaultKmsKeyName; /** * Base class for bucket's delete rules. Allows to configure automatic deletion of blobs and blobs @@ -423,6 +425,11 @@ public abstract static class Builder { */ public abstract Builder setLabels(Map labels); + /** + * Sets the default Cloud KMS key name for this bucket. + */ + public abstract Builder setDefaultKmsKeyName(String defaultKmsKeyName); + /** * Creates a {@code BucketInfo} object. */ @@ -449,6 +456,7 @@ static final class BuilderImpl extends Builder { private List acl; private List defaultAcl; private Map labels; + private String defaultKmsKeyName; BuilderImpl(String name) { this.name = name; @@ -473,6 +481,7 @@ static final class BuilderImpl extends Builder { deleteRules = bucketInfo.deleteRules; labels = bucketInfo.labels; requesterPays = bucketInfo.requesterPays; + defaultKmsKeyName = bucketInfo.defaultKmsKeyName; } @Override @@ -584,6 +593,12 @@ public Builder setLabels(Map labels) { return this; } + @Override + public Builder setDefaultKmsKeyName(String defaultKmsKeyName) { + this.defaultKmsKeyName = defaultKmsKeyName; + return this; + } + @Override public BucketInfo build() { checkNotNull(name); @@ -610,6 +625,7 @@ public BucketInfo build() { deleteRules = builder.deleteRules; labels = builder.labels; requesterPays = builder.requesterPays; + defaultKmsKeyName = builder.defaultKmsKeyName; } /** @@ -762,6 +778,13 @@ public Map getLabels() { return labels; } + /** + * Returns the default Cloud KMS key to be applied to newly inserted objects in this bucket. + */ + public String getDefaultKmsKeyName() { + return defaultKmsKeyName; + } + /** * Returns a builder for the current bucket. */ @@ -857,7 +880,9 @@ public Rule apply(DeleteRule deleteRule) { if (labels != null) { bucketPb.setLabels(labels); } - + if (defaultKmsKeyName != null) { + bucketPb.setEncryption(new Encryption().setDefaultKmsKeyName(defaultKmsKeyName)); + } return bucketPb; } @@ -945,6 +970,10 @@ public DeleteRule apply(Rule rule) { if (billing != null) { builder.setRequesterPays(billing.getRequesterPays()); } + Encryption encryption = bucketPb.getEncryption(); + if (encryption != null && encryption.getDefaultKmsKeyName() != null && !encryption.getDefaultKmsKeyName().isEmpty()) { + builder.setDefaultKmsKeyName(encryption.getDefaultKmsKeyName()); + } return builder.build(); } } From 56b34dfb1ac66b12ba8c3d13aa9147501e8692b6 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Tue, 1 May 2018 01:14:00 -0700 Subject: [PATCH 02/13] Adding Integration tests and partial unit test coverage --- .../java/com/google/cloud/storage/Blob.java | 5 +- .../com/google/cloud/storage/BlobInfo.java | 28 +++-- .../java/com/google/cloud/storage/Bucket.java | 3 - .../com/google/cloud/storage/BucketInfo.java | 6 +- .../com/google/cloud/storage/Storage.java | 18 +-- .../google/cloud/storage/BlobInfoTest.java | 7 ++ .../google/cloud/storage/BucketInfoTest.java | 4 + .../com/google/cloud/storage/BucketTest.java | 2 + .../cloud/storage/it/ITStorageTest.java | 119 ++++++++++++++++++ 9 files changed, 155 insertions(+), 37 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java index 14959a2c58ca..0f6a8e379485 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java @@ -184,7 +184,6 @@ public static BlobSourceOption decryptionKey(String key) { * Returns an option for blob's billing user project. This option is used only if the blob's * bucket has requester_pays flag enabled. */ - @GcpLaunchStage.Alpha public static BlobSourceOption userProject(String userProject) { return new BlobSourceOption(StorageRpc.Option.USER_PROJECT, userProject); } @@ -404,8 +403,8 @@ Builder setCustomerEncryption(CustomerEncryption customerEncryption) { } @Override - Builder setKmsKey(String kmsKey) { - infoBuilder.setKmsKey(kmsKey); + public Builder setKmsKeyName(String kmsKeyName) { + infoBuilder.setKmsKeyName(kmsKeyName); return this; } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java index b5f6674046e8..85df1f6f150e 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java @@ -83,7 +83,7 @@ public StorageObject apply(BlobInfo blobInfo) { private final Integer componentCount; private final boolean isDirectory; private final CustomerEncryption customerEncryption; - private final String kmsKey; + private final String kmsKeyName; /** * This class is meant for internal use only. Users are discouraged from using this class. @@ -267,7 +267,11 @@ public abstract static class Builder { abstract Builder setCustomerEncryption(CustomerEncryption customerEncryption); - abstract Builder setKmsKey(String kmsKey); + /** + * + * Sets the blob's kmsKeyName. + */ + public abstract Builder setKmsKeyName(String kmsKeyName); /** * Creates a {@code BlobInfo} object. @@ -301,7 +305,7 @@ static final class BuilderImpl extends Builder { private Boolean isDirectory; private CustomerEncryption customerEncryption; private StorageClass storageClass; - private String kmsKey; + private String kmsKeyName; BuilderImpl(BlobId blobId) { this.blobId = blobId; @@ -332,7 +336,7 @@ static final class BuilderImpl extends Builder { createTime = blobInfo.createTime; isDirectory = blobInfo.isDirectory; storageClass = blobInfo.storageClass; - kmsKey = blobInfo.kmsKey; + kmsKeyName = blobInfo.kmsKeyName; } @Override @@ -481,8 +485,8 @@ Builder setCustomerEncryption(CustomerEncryption customerEncryption) { } @Override - Builder setKmsKey(String kmsKey) { - this.kmsKey = kmsKey; + public Builder setKmsKeyName(String kmsKeyName) { + this.kmsKeyName = kmsKeyName; return this; } @@ -518,7 +522,7 @@ public BlobInfo build() { createTime = builder.createTime; isDirectory = firstNonNull(builder.isDirectory, Boolean.FALSE); storageClass = builder.storageClass; - kmsKey = builder.kmsKey; + kmsKeyName = builder.kmsKeyName; } /** @@ -752,8 +756,8 @@ public StorageClass getStorageClass() { /** * Returns the Cloud KMS key used to encrypt the blob, if any. */ - public String getKmsKey() { - return kmsKey; + public String getKmsKeyName() { + return kmsKeyName; } /** @@ -828,8 +832,8 @@ public ObjectAccessControl apply(Acl acl) { if (customerEncryption != null) { storageObject.setCustomerEncryption(customerEncryption.toPb()); } - if (kmsKey != null) { - storageObject.setKmsKeyName(kmsKey); + if (kmsKeyName != null) { + storageObject.setKmsKeyName(kmsKeyName); } storageObject.setMetadata(pbMetadata); storageObject.setCacheControl(cacheControl); @@ -962,7 +966,7 @@ public Acl apply(ObjectAccessControl objectAccessControl) { builder.setStorageClass(StorageClass.valueOf(storageObject.getStorageClass())); } if (storageObject.getKmsKeyName() != null) { - builder.setKmsKey(storageObject.getKmsKeyName()); + builder.setKmsKeyName(storageObject.getKmsKeyName()); } return builder.build(); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java index 9b07f4567f1a..07254e1185f6 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java @@ -118,7 +118,6 @@ public static BucketSourceOption metagenerationNotMatch() { * Returns an option for blob's billing user project. This option is only used by the buckets with * 'requester_pays' flag. */ - @GcpLaunchStage.Alpha public static BucketSourceOption userProject(String userProject) { return new BucketSourceOption(StorageRpc.Option.USER_PROJECT, userProject); } @@ -271,7 +270,6 @@ public static BlobTargetOption encryptionKey(String key) { * Returns an option for blob's billing user project. This option is only used by the buckets with * 'requester_pays' flag. */ - @GcpLaunchStage.Alpha public static BlobTargetOption userProject(String userProject) { return new BlobTargetOption(StorageRpc.Option.USER_PROJECT, userProject); } @@ -468,7 +466,6 @@ public static BlobWriteOption encryptionKey(String key) { * Returns an option for blob's billing user project. This option is only used by the buckets with * 'requester_pays' flag. */ - @GcpLaunchStage.Alpha public static BlobWriteOption userProject(String userProject) { return new BlobWriteOption(Storage.BlobWriteOption.Option.USER_PROJECT, userProject); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java index bfdce530aac0..79b6a89b2830 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java @@ -880,9 +880,9 @@ public Rule apply(DeleteRule deleteRule) { if (labels != null) { bucketPb.setLabels(labels); } - if (defaultKmsKeyName != null) { - bucketPb.setEncryption(new Encryption().setDefaultKmsKeyName(defaultKmsKeyName)); - } + // default kms key name can be null. + bucketPb.setEncryption(new Encryption().setDefaultKmsKeyName(defaultKmsKeyName)); + return bucketPb; } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 728438145493..232051fd8e99 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -94,7 +94,7 @@ enum BucketField implements FieldSelector { CORS("cors"), STORAGE_CLASS("storageClass"), ETAG("etag"), - @GcpLaunchStage.Alpha + DEFAULT_KMS_KEY_NAME("defaultKmsKeyName"), BILLING("billing"); static final List REQUIRED_FIELDS = ImmutableList.of(NAME); @@ -136,6 +136,7 @@ enum BlobField implements FieldSelector { SIZE("size"), STORAGE_CLASS("storageClass"), TIME_DELETED("timeDeleted"), + KMS_KEY_NAME("kmsKeyName"), UPDATED("updated"); static final List REQUIRED_FIELDS = ImmutableList.of(BUCKET, NAME); @@ -203,7 +204,6 @@ public static BucketTargetOption metagenerationNotMatch() { * Returns an option to define the billing user project. This option is required by buckets with * `requester_pays` flag enabled to assign operation costs. */ - @GcpLaunchStage.Alpha public static BucketTargetOption userProject(String userProject) { return new BucketTargetOption(StorageRpc.Option.USER_PROJECT, userProject); } @@ -240,7 +240,6 @@ public static BucketSourceOption metagenerationNotMatch(long metageneration) { * Returns an option for bucket's billing user project. This option is only used by the buckets with * 'requester_pays' flag. */ - @GcpLaunchStage.Alpha public static BucketSourceOption userProject(String userProject) { return new BucketSourceOption(StorageRpc.Option.USER_PROJECT, userProject); } @@ -281,7 +280,6 @@ public static BucketGetOption metagenerationNotMatch(long metageneration) { * Returns an option for bucket's billing user project. This option is only used by the buckets with * 'requester_pays' flag. */ - @GcpLaunchStage.Alpha public static BucketGetOption userProject(String userProject) { return new BucketGetOption(StorageRpc.Option.USER_PROJECT, userProject); } @@ -372,7 +370,6 @@ public static BlobTargetOption encryptionKey(Key key) { * Returns an option for blob's billing user project. This option is only used by the buckets with * 'requester_pays' flag. */ - @GcpLaunchStage.Alpha public static BlobTargetOption userProject(String userProject) { return new BlobTargetOption(StorageRpc.Option.USER_PROJECT, userProject); } @@ -542,7 +539,6 @@ public static BlobWriteOption encryptionKey(String key) { * Returns an option for blob's billing user project. This option is only used by the buckets with * 'requester_pays' flag. */ - @GcpLaunchStage.Alpha public static BlobWriteOption userProject(String userProject) { return new BlobWriteOption(Option.USER_PROJECT, userProject); } @@ -636,7 +632,6 @@ public static BlobSourceOption decryptionKey(String key) { * Returns an option for blob's billing user project. This option is only used by the buckets with * 'requester_pays' flag. */ - @GcpLaunchStage.Alpha public static BlobSourceOption userProject(String userProject) { return new BlobSourceOption(StorageRpc.Option.USER_PROJECT, userProject); } @@ -726,7 +721,6 @@ public static BlobGetOption fields(BlobField... fields) { * Returns an option for blob's billing user project. This option is only used by the buckets with * 'requester_pays' flag. */ - @GcpLaunchStage.Alpha public static BlobGetOption userProject(String userProject) { return new BlobGetOption(StorageRpc.Option.USER_PROJECT, userProject); } @@ -769,7 +763,6 @@ public static BucketListOption prefix(String prefix) { * Returns an option for bucket's billing user project. This option is only used by the buckets with * 'requester_pays' flag. */ - @GcpLaunchStage.Alpha public static BucketListOption userProject(String userProject) { return new BucketListOption(StorageRpc.Option.USER_PROJECT, userProject); } @@ -840,7 +833,6 @@ public static BlobListOption currentDirectory() { * * @param userProject projectId of the billing user project. */ - @GcpLaunchStage.Alpha public static BlobListOption userProject(String userProject) { return new BlobListOption(StorageRpc.Option.USER_PROJECT, userProject); } @@ -2587,8 +2579,6 @@ public static Builder newBuilder() { * @param options extra parameters to apply to this operation * @throws StorageException upon failure */ - @BetaApi - @GcpLaunchStage.Alpha Policy getIamPolicy(String bucket, BucketSourceOption... options); /** @@ -2612,8 +2602,6 @@ public static Builder newBuilder() { * @param options extra parameters to apply to this operation * @throws StorageException upon failure */ - @BetaApi - @GcpLaunchStage.Alpha Policy setIamPolicy(String bucket, Policy policy, BucketSourceOption... options); /** @@ -2637,8 +2625,6 @@ public static Builder newBuilder() { * @param options extra parameters to apply to this operation * @throws StorageException upon failure */ - @BetaApi - @GcpLaunchStage.Alpha List testIamPermissions(String bucket, List permissions, BucketSourceOption... options); /** diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobInfoTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobInfoTest.java index 16accd0e4a93..508c03574815 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobInfoTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobInfoTest.java @@ -66,8 +66,11 @@ public class BlobInfoTest { private static final String KEY_SHA256 = "keySha"; private static final CustomerEncryption CUSTOMER_ENCRYPTION = new CustomerEncryption(ENCRYPTION_ALGORITHM, KEY_SHA256); + private static final String KMS_KEY_NAME = "projects/p/keyRings/kr/location/kr-loc/cryptoKey/key"; private static final StorageClass STORAGE_CLASS = StorageClass.COLDLINE; + // Unit tests include Customer Supplied encryption which may be confusing with the KMS KEY NAME + // as both can't be set at the same time. At least not right now. private static final BlobInfo BLOB_INFO = BlobInfo.newBuilder("b", "n", GENERATION) .setAcl(ACL) .setComponentCount(COMPONENT_COUNT) @@ -91,6 +94,7 @@ public class BlobInfoTest { .setUpdateTime(UPDATE_TIME) .setCreateTime(CREATE_TIME) .setStorageClass(STORAGE_CLASS) + .setKmsKeyName(KMS_KEY_NAME) .build(); private static final BlobInfo DIRECTORY_INFO = BlobInfo.newBuilder("b", "n/") .setSize(0L) @@ -153,6 +157,7 @@ public void testBuilder() { assertEquals(UPDATE_TIME, BLOB_INFO.getUpdateTime()); assertEquals(CREATE_TIME, BLOB_INFO.getCreateTime()); assertEquals(STORAGE_CLASS, BLOB_INFO.getStorageClass()); + assertEquals(KMS_KEY_NAME, BLOB_INFO.getKmsKeyName()); assertFalse(BLOB_INFO.isDirectory()); assertEquals("b", DIRECTORY_INFO.getBucket()); assertEquals("n/", DIRECTORY_INFO.getName()); @@ -208,6 +213,7 @@ private void compareBlobs(BlobInfo expected, BlobInfo value) { assertEquals(expected.getSize(), value.getSize()); assertEquals(expected.getUpdateTime(), value.getUpdateTime()); assertEquals(expected.getStorageClass(), value.getStorageClass()); + assertEquals(expected.getKmsKeyName(), value.getKmsKeyName()); } private void compareCustomerEncryptions(CustomerEncryption expected, CustomerEncryption value) { @@ -255,6 +261,7 @@ public void testToPbAndFromPb() { assertEquals(0L, (long) blobInfo.getSize()); assertNull(blobInfo.getUpdateTime()); assertNull(blobInfo.getStorageClass()); + assertNull(blobInfo.getKmsKeyName()); assertTrue(blobInfo.isDirectory()); } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketInfoTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketInfoTest.java index 49010ef181db..117673e51943 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketInfoTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketInfoTest.java @@ -60,6 +60,7 @@ public class BucketInfoTest { private static final String NOT_FOUND_PAGE = "error.html"; private static final String LOCATION = "ASIA"; private static final StorageClass STORAGE_CLASS = StorageClass.STANDARD; + private static final String DEFAULT_KMS_KEY_NAME = "projects/p/keyRings/kr/location/kr-loc/cryptoKey/key"; private static final Boolean VERSIONING_ENABLED = true; private static final Map BUCKET_LABELS = ImmutableMap.of("label1", "value1"); private static final Boolean REQUESTER_PAYS = true; @@ -81,6 +82,7 @@ public class BucketInfoTest { .setVersioningEnabled(VERSIONING_ENABLED) .setLabels(BUCKET_LABELS) .setRequesterPays(REQUESTER_PAYS) + .setDefaultKmsKeyName(DEFAULT_KMS_KEY_NAME) .build(); @Test @@ -122,6 +124,7 @@ public void testBuilder() { assertEquals(NOT_FOUND_PAGE, BUCKET_INFO.getNotFoundPage()); assertEquals(LOCATION, BUCKET_INFO.getLocation()); assertEquals(STORAGE_CLASS, BUCKET_INFO.getStorageClass()); + assertEquals(DEFAULT_KMS_KEY_NAME, BUCKET_INFO.getDefaultKmsKeyName()); assertEquals(VERSIONING_ENABLED, BUCKET_INFO.versioningEnabled()); assertEquals(BUCKET_LABELS, BUCKET_INFO.getLabels()); assertEquals(REQUESTER_PAYS, BUCKET_INFO.requesterPays()); @@ -151,6 +154,7 @@ private void compareBuckets(BucketInfo expected, BucketInfo value) { assertEquals(expected.getNotFoundPage(), value.getNotFoundPage()); assertEquals(expected.getLocation(), value.getLocation()); assertEquals(expected.getStorageClass(), value.getStorageClass()); + assertEquals(expected.getDefaultKmsKeyName(), value.getDefaultKmsKeyName()); assertEquals(expected.versioningEnabled(), value.versioningEnabled()); assertEquals(expected.getLabels(), value.getLabels()); assertEquals(expected.requesterPays(), value.requesterPays()); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java index 2b46316f3614..05edb3b26af5 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java @@ -77,6 +77,7 @@ public class BucketTest { private static final String NOT_FOUND_PAGE = "error.html"; private static final String LOCATION = "ASIA"; private static final StorageClass STORAGE_CLASS = StorageClass.STANDARD; + private static final String DEFAULT_KMS_KEY_NAME = "projects/p/keyRings/kr/location/kr-loc/cryptoKey/key"; private static final Boolean VERSIONING_ENABLED = true; private static final Map BUCKET_LABELS = ImmutableMap.of("label1", "value1"); private static final Boolean REQUESTER_PAYS = true; @@ -96,6 +97,7 @@ public class BucketTest { .setNotFoundPage(NOT_FOUND_PAGE) .setLocation(LOCATION) .setStorageClass(STORAGE_CLASS) + .setDefaultKmsKeyName(DEFAULT_KMS_KEY_NAME) .setVersioningEnabled(VERSIONING_ENABLED) .setLabels(BUCKET_LABELS) .setRequesterPays(REQUESTER_PAYS) diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index 9f6272b87bf2..3ebd0150e365 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -108,6 +108,7 @@ public class ITStorageTest { .decode("H4sIAAAAAAAAAPNIzcnJV3DPz0/PSVVwzskvTVEILskvSkxPVQQA/LySchsAAAA="); private static final Map BUCKET_LABELS = ImmutableMap.of("label1", "value1"); private static final String SERVICE_ACCOUNT_EMAIL = "gcloud-devel@gs-project-accounts.iam.gserviceaccount.com"; + private static final String KMS_KEY_NAME = ""; // use real KMS KEY associated to the test project. @BeforeClass public static void beforeClass() throws NoSuchAlgorithmException, InvalidKeySpecException { @@ -177,6 +178,21 @@ public void testGetBucketEmptyFields() { assertNull(remoteBucket.getSelfLink()); } + @Test + public void testClearBucketDefaultKmsKeyName() throws ExecutionException, InterruptedException { + String bucketName = RemoteStorageHelper.generateBucketName(); + Bucket remoteBucket = storage.create(BucketInfo.newBuilder(bucketName).setDefaultKmsKeyName(KMS_KEY_NAME).build()); + + try { + assertEquals(KMS_KEY_NAME, remoteBucket.getDefaultKmsKeyName()); + remoteBucket = remoteBucket.toBuilder().setDefaultKmsKeyName(null).build(); + Bucket updatedBucket = storage.update(remoteBucket); + assertEquals(null, updatedBucket.getDefaultKmsKeyName()); + } finally { + RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); + } + } + @Test public void testCreateBlob() { String blobName = "test-create-blob"; @@ -204,6 +220,39 @@ public void testCreateBlobWithEncryptionKey() { assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); } + @Test + public void testCreateBlobWithKmsKeyName() { + String blobName = "test-create-with-kms-key-name-blob"; + BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).setKmsKeyName(KMS_KEY_NAME).build(); + Blob remoteBlob = storage.create(blob, BLOB_BYTE_CONTENT); + assertNotNull(remoteBlob); + assertEquals(blob.getBucket(), remoteBlob.getBucket()); + assertEquals(blob.getName(), remoteBlob.getName()); + assertEquals(blob.getKmsKeyName(), remoteBlob.getKmsKeyName()); + byte[] readBytes = storage.readAllBytes(BUCKET, blobName); + assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); + } + + @Test + public void testCreateBlobWithDefaultKmsKeyName() throws ExecutionException, InterruptedException { + String bucketName = RemoteStorageHelper.generateBucketName(); + Bucket bucket = storage.create(BucketInfo.newBuilder(bucketName).setDefaultKmsKeyName(KMS_KEY_NAME).build()); + + try { + String blobName = "test-create-with-default-kms-key-name-blob"; + BlobInfo blob = BlobInfo.newBuilder(bucket, blobName).build(); + Blob remoteBlob = storage.create(blob, BLOB_BYTE_CONTENT); + assertNotNull(remoteBlob); + assertEquals(blob.getBucket(), remoteBlob.getBucket()); + assertEquals(blob.getName(), remoteBlob.getName()); + assertEquals(blob.getKmsKeyName(), remoteBlob.getKmsKeyName()); + byte[] readBytes = storage.readAllBytes(BUCKET, blobName); + assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); + } finally { + RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); + } + } + @Test public void testCreateEmptyBlob() { String blobName = "test-create-empty-blob"; @@ -373,6 +422,44 @@ public void testListBlobsSelectedFields() throws InterruptedException { } } + @Test(timeout = 5000) + public void testListBlobsKmsKeySelectedFields() throws InterruptedException { + String[] blobNames = {"test-list-blobs-selected-field-kms-key-name-blob1", + "test-list-blobs-selected-field-kms-key-name-blob2"}; + BlobInfo blob1 = BlobInfo.newBuilder(BUCKET, blobNames[0]) + .setContentType(CONTENT_TYPE) + .setKmsKeyName(KMS_KEY_NAME) + .build(); + BlobInfo blob2 = BlobInfo.newBuilder(BUCKET, blobNames[1]) + .setContentType(CONTENT_TYPE) + .setKmsKeyName(KMS_KEY_NAME) + .build(); + Blob remoteBlob1 = storage.create(blob1); + Blob remoteBlob2 = storage.create(blob2); + assertNotNull(remoteBlob1); + assertNotNull(remoteBlob2); + Page page = storage.list(BUCKET, + Storage.BlobListOption.prefix("test-list-blobs-selected-field-kms-key-name-blob"), + Storage.BlobListOption.fields(BlobField.KMS_KEY_NAME)); + // Listing blobs is eventually consistent, we loop until the list is of the expected size. The + // test fails if timeout is reached. + while (Iterators.size(page.iterateAll().iterator()) != 2) { + Thread.sleep(500); + page = storage.list(BUCKET, + Storage.BlobListOption.prefix("test-list-blobs-selected-field-kms-key-name-blob"), + Storage.BlobListOption.fields(BlobField.KMS_KEY_NAME)); + } + Set blobSet = ImmutableSet.of(blobNames[0], blobNames[1]); + Iterator iterator = page.iterateAll().iterator(); + while (iterator.hasNext()) { + Blob remoteBlob = iterator.next(); + assertEquals(BUCKET, remoteBlob.getBucket()); + assertTrue(blobSet.contains(remoteBlob.getName())); + assertEquals(KMS_KEY_NAME, remoteBlob.getKmsKeyName()); + assertNull(remoteBlob.getContentType()); + } + } + @Test(timeout = 5000) public void testListBlobsEmptySelectedFields() throws InterruptedException { String[] blobNames = {"test-list-blobs-empty-selected-fields-blob1", @@ -452,6 +539,8 @@ public void testListBlobRequesterPays() throws InterruptedException { } } + + @Test(timeout = 15000) public void testListBlobsVersioned() throws ExecutionException, InterruptedException { String bucketName = RemoteStorageHelper.generateBucketName(); @@ -806,6 +895,36 @@ public void testCopyBlobWithEncryptionKeys() { assertTrue(storage.delete(BUCKET, targetBlobName)); } + @Test + public void testRotateFromCustomerEncryptionToKmsKey() { + String sourceBlobName = "test-copy-blob-encryption-key-source"; + BlobId source = BlobId.of(BUCKET, sourceBlobName); + ImmutableMap metadata = ImmutableMap.of("k", "v"); + Blob remoteBlob = storage.create(BlobInfo.newBuilder(source).build(), BLOB_BYTE_CONTENT, + Storage.BlobTargetOption.encryptionKey(KEY)); + assertNotNull(remoteBlob); + String targetBlobName = "test-copy-blob-kms-key-target"; + BlobInfo target = BlobInfo.newBuilder(BUCKET, targetBlobName) + .setContentType(CONTENT_TYPE) + .setMetadata(metadata) + .setKmsKeyName(KMS_KEY_NAME) + .build(); + Storage.CopyRequest req = Storage.CopyRequest.newBuilder() + .setSource(source) + .setSourceOptions(Storage.BlobSourceOption.decryptionKey(BASE64_KEY)) + .setTarget(target) + .build(); + CopyWriter copyWriter = storage.copy(req); + assertEquals(BUCKET, copyWriter.getResult().getBucket()); + assertEquals(targetBlobName, copyWriter.getResult().getName()); + assertEquals(CONTENT_TYPE, copyWriter.getResult().getContentType()); + assertEquals(KMS_KEY_NAME, copyWriter.getResult().getKmsKeyName()); + assertArrayEquals(BLOB_BYTE_CONTENT, copyWriter.getResult().getContent()); + assertEquals(metadata, copyWriter.getResult().getMetadata()); + assertTrue(copyWriter.isDone()); + assertTrue(storage.delete(BUCKET, targetBlobName)); + } + @Test public void testCopyBlobUpdateMetadata() { String sourceBlobName = "test-copy-blob-update-metadata-source"; From 47701ed4c644a4c19b99ebe8aaef1e896b533f0d Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Tue, 1 May 2018 01:42:34 -0700 Subject: [PATCH 03/13] Fix kms key reosurce name --- .../src/test/java/com/google/cloud/storage/BlobInfoTest.java | 2 +- .../src/test/java/com/google/cloud/storage/BucketInfoTest.java | 2 +- .../src/test/java/com/google/cloud/storage/BucketTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobInfoTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobInfoTest.java index 508c03574815..3160af0cc8da 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobInfoTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobInfoTest.java @@ -66,7 +66,7 @@ public class BlobInfoTest { private static final String KEY_SHA256 = "keySha"; private static final CustomerEncryption CUSTOMER_ENCRYPTION = new CustomerEncryption(ENCRYPTION_ALGORITHM, KEY_SHA256); - private static final String KMS_KEY_NAME = "projects/p/keyRings/kr/location/kr-loc/cryptoKey/key"; + private static final String KMS_KEY_NAME = "projects/p/locations/kr-loc/keyRings/kr/cryptoKeys/key"; private static final StorageClass STORAGE_CLASS = StorageClass.COLDLINE; // Unit tests include Customer Supplied encryption which may be confusing with the KMS KEY NAME diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketInfoTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketInfoTest.java index 117673e51943..d31dd0246e0b 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketInfoTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketInfoTest.java @@ -60,7 +60,7 @@ public class BucketInfoTest { private static final String NOT_FOUND_PAGE = "error.html"; private static final String LOCATION = "ASIA"; private static final StorageClass STORAGE_CLASS = StorageClass.STANDARD; - private static final String DEFAULT_KMS_KEY_NAME = "projects/p/keyRings/kr/location/kr-loc/cryptoKey/key"; + private static final String DEFAULT_KMS_KEY_NAME = "projects/p/locations/kr-loc/keyRings/kr/cryptoKeys/key"; private static final Boolean VERSIONING_ENABLED = true; private static final Map BUCKET_LABELS = ImmutableMap.of("label1", "value1"); private static final Boolean REQUESTER_PAYS = true; diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java index 05edb3b26af5..34c14781857a 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java @@ -77,7 +77,7 @@ public class BucketTest { private static final String NOT_FOUND_PAGE = "error.html"; private static final String LOCATION = "ASIA"; private static final StorageClass STORAGE_CLASS = StorageClass.STANDARD; - private static final String DEFAULT_KMS_KEY_NAME = "projects/p/keyRings/kr/location/kr-loc/cryptoKey/key"; + private static final String DEFAULT_KMS_KEY_NAME = "projects/p/locations/kr-loc/keyRings/kr/cryptoKeys/key"; private static final Boolean VERSIONING_ENABLED = true; private static final Map BUCKET_LABELS = ImmutableMap.of("label1", "value1"); private static final Boolean REQUESTER_PAYS = true; From 86830e59f66d58dcd6a9552510509561a006cf62 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Sun, 6 May 2018 23:06:21 -0700 Subject: [PATCH 04/13] Intermediate stage with WriteOptions --- google-cloud-storage/pom.xml | 1 + .../com/google/cloud/storage/BlobInfo.java | 5 +- .../java/com/google/cloud/storage/Bucket.java | 19 +++++ .../com/google/cloud/storage/BucketInfo.java | 9 +-- .../com/google/cloud/storage/Storage.java | 27 ++++++- .../com/google/cloud/storage/StorageImpl.java | 2 + .../cloud/storage/spi/v1/HttpStorageRpc.java | 10 ++- .../cloud/storage/spi/v1/StorageRpc.java | 3 +- .../google/cloud/storage/BlobInfoTest.java | 2 - .../com/google/cloud/storage/BlobTest.java | 1 + .../com/google/cloud/storage/BucketTest.java | 17 +++-- .../google/cloud/storage/StorageImplTest.java | 2 + .../cloud/storage/it/ITStorageTest.java | 71 ++++++++++++++----- 13 files changed, 133 insertions(+), 36 deletions(-) diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index c88f3ba7ba30..d7765f318844 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -29,6 +29,7 @@ com.google.apis google-api-services-storage + v1-rev125-1.23.0 compile diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java index 85df1f6f150e..71e711ee95c7 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java @@ -832,9 +832,8 @@ public ObjectAccessControl apply(Acl acl) { if (customerEncryption != null) { storageObject.setCustomerEncryption(customerEncryption.toPb()); } - if (kmsKeyName != null) { - storageObject.setKmsKeyName(kmsKeyName); - } + + storageObject.setKmsKeyName(kmsKeyName); storageObject.setMetadata(pbMetadata); storageObject.setCacheControl(cacheControl); storageObject.setContentEncoding(contentEncoding); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java index 07254e1185f6..c5a399391de9 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java @@ -190,6 +190,9 @@ private Tuple toTargetOption(BlobInfo blobIn case USER_PROJECT: return Tuple.of(blobInfo, Storage.BlobTargetOption.userProject((String) getValue())); + case KMS_KEY_NAME: + return Tuple.of(blobInfo, + Storage.BlobTargetOption.kmsKeyName((String) getValue())); default: throw new AssertionError("Unexpected enum value"); } @@ -274,6 +277,13 @@ public static BlobTargetOption userProject(String userProject) { return new BlobTargetOption(StorageRpc.Option.USER_PROJECT, userProject); } + /** + * Returns an option for blob's kms key name. + */ + public static BlobTargetOption kmsKeyName(String kmsKeyName) { + return new BlobTargetOption(StorageRpc.Option.KMS_KEY_NAME, kmsKeyName); + } + static Tuple toTargetOptions( BlobInfo info, BlobTargetOption... options) { Set optionSet = @@ -345,6 +355,8 @@ private Tuple toWriteOption(BlobInfo blobInfo Storage.BlobWriteOption.encryptionKey((String) value)); case USER_PROJECT: return Tuple.of(blobInfo, Storage.BlobWriteOption.userProject((String) value)); + case KMS_KEY_NAME: + return Tuple.of(blobInfo, Storage.BlobWriteOption.kmsKeyName((String) value)); default: throw new AssertionError("Unexpected enum value"); } @@ -470,6 +482,13 @@ public static BlobWriteOption userProject(String userProject) { return new BlobWriteOption(Storage.BlobWriteOption.Option.USER_PROJECT, userProject); } + /** + * Returns an option for blob's kms key name. + */ + public static BlobWriteOption kmsKeyName(String kmsKeyName) { + return new BlobWriteOption(Storage.BlobWriteOption.Option.KMS_KEY_NAME, kmsKeyName); + } + static Tuple toWriteOptions( BlobInfo info, BlobWriteOption... options) { Set optionSet = diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java index 79b6a89b2830..8a0a24ef610e 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java @@ -595,7 +595,8 @@ public Builder setLabels(Map labels) { @Override public Builder setDefaultKmsKeyName(String defaultKmsKeyName) { - this.defaultKmsKeyName = defaultKmsKeyName; + this.defaultKmsKeyName = defaultKmsKeyName != null + ? new String(defaultKmsKeyName) : Data.nullOf(String.class); return this; } @@ -880,9 +881,9 @@ public Rule apply(DeleteRule deleteRule) { if (labels != null) { bucketPb.setLabels(labels); } - // default kms key name can be null. - bucketPb.setEncryption(new Encryption().setDefaultKmsKeyName(defaultKmsKeyName)); - + if (defaultKmsKeyName != null) { + bucketPb.setEncryption(new Encryption().setDefaultKmsKeyName(defaultKmsKeyName)); + } return bucketPb; } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 232051fd8e99..8d6e85c06204 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import com.google.api.client.util.Data; import com.google.api.core.BetaApi; import com.google.api.gax.paging.Page; import com.google.auth.ServiceAccountSigner; @@ -94,7 +95,7 @@ enum BucketField implements FieldSelector { CORS("cors"), STORAGE_CLASS("storageClass"), ETAG("etag"), - DEFAULT_KMS_KEY_NAME("defaultKmsKeyName"), + ENCRYPTION("encryption"), BILLING("billing"); static final List REQUIRED_FIELDS = ImmutableList.of(NAME); @@ -207,6 +208,15 @@ public static BucketTargetOption metagenerationNotMatch() { public static BucketTargetOption userProject(String userProject) { return new BucketTargetOption(StorageRpc.Option.USER_PROJECT, userProject); } + +// /** +// * Returns an option to define the defaultKmsKeyName. +// */ +// public static BucketTargetOption defaultKmsKeyName(String defaultKmsKeyName) { +// defaultKmsKeyName = defaultKmsKeyName != null +// ? new String(defaultKmsKeyName) : Data.nullOf(String.class); +// return new BucketTargetOption(StorageRpc.Option.KMS_KEY_NAME, defaultKmsKeyName); +// } } /** @@ -384,6 +394,15 @@ public static BlobTargetOption encryptionKey(String key) { return new BlobTargetOption(StorageRpc.Option.CUSTOMER_SUPPLIED_KEY, key); } + /** + * Returns an option to set a KMS key resource name of the blob. + * + * @param kmsKeyName the KMS crypto key resource name + */ + public static BlobTargetOption kmsKeyName(String kmsKeyName) { + return new BlobTargetOption(StorageRpc.Option.KMS_KEY_NAME, kmsKeyName); + } + static Tuple convert(BlobInfo info, BlobWriteOption... options) { BlobInfo.Builder infoBuilder = info.toBuilder().setCrc32c(null).setMd5(null); List targetOptions = Lists.newArrayListWithCapacity(options.length); @@ -417,7 +436,7 @@ class BlobWriteOption implements Serializable { enum Option { PREDEFINED_ACL, IF_GENERATION_MATCH, IF_GENERATION_NOT_MATCH, IF_METAGENERATION_MATCH, - IF_METAGENERATION_NOT_MATCH, IF_MD5_MATCH, IF_CRC32C_MATCH, CUSTOMER_SUPPLIED_KEY, USER_PROJECT; + IF_METAGENERATION_NOT_MATCH, IF_MD5_MATCH, IF_CRC32C_MATCH, CUSTOMER_SUPPLIED_KEY, USER_PROJECT, KMS_KEY_NAME; StorageRpc.Option toRpcOption() { return StorageRpc.Option.valueOf(this.name()); @@ -542,6 +561,10 @@ public static BlobWriteOption encryptionKey(String key) { public static BlobWriteOption userProject(String userProject) { return new BlobWriteOption(Option.USER_PROJECT, userProject); } + + public static BlobWriteOption kmsKeyName(String kmsKeyName) { + return new BlobWriteOption(Option.KMS_KEY_NAME, kmsKeyName); + } } /** diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index e49445d13646..20ebab1f2174 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -437,9 +437,11 @@ public CopyWriter copy(final CopyRequest copyRequest) { RewriteResponse rewriteResponse = runWithRetries(new Callable() { @Override public RewriteResponse call() { + return storageRpc.openRewrite(new StorageRpc.RewriteRequest(source, sourceOptions, copyRequest.overrideInfo(), targetObject, targetOptions, copyRequest.getMegabytesCopiedPerChunk())); + } }, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock()); return new CopyWriter(getOptions(), rewriteResponse); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index b0a71387690a..19b03d113a5f 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -426,6 +426,7 @@ public StorageObject get(StorageObject object, Map options) { public Bucket patch(Bucket bucket, Map options) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_PATCH_BUCKET); Scope scope = tracer.withSpan(span); + try { return storage.buckets() .patch(bucket.getName(), bucket) @@ -790,6 +791,11 @@ private RewriteResponse rewrite(RewriteRequest req, String token) { userProject = Option.USER_PROJECT.getString(req.targetOptions); } + String kmsKeyName = Option.KMS_KEY_NAME.getString(req.targetOptions); + if (kmsKeyName == null) { + kmsKeyName = req.target.getKmsKeyName(); + } + Long maxBytesRewrittenPerCall = req.megabytesRewrittenPerCall != null ? req.megabytesRewrittenPerCall * MEGABYTE : null; Storage.Objects.Rewrite rewrite = storage.objects() @@ -808,7 +814,9 @@ private RewriteResponse rewrite(RewriteRequest req, String token) { .setIfMetagenerationNotMatch(Option.IF_METAGENERATION_NOT_MATCH.getLong(req.targetOptions)) .setIfGenerationMatch(Option.IF_GENERATION_MATCH.getLong(req.targetOptions)) .setIfGenerationNotMatch(Option.IF_GENERATION_NOT_MATCH.getLong(req.targetOptions)) - .setUserProject(userProject); + .setUserProject(userProject) + .setDestinationKmsKeyName(kmsKeyName); + HttpHeaders requestHeaders = rewrite.getRequestHeaders(); setEncryptionHeaders(requestHeaders, SOURCE_ENCRYPTION_KEY_PREFIX, req.sourceOptions); setEncryptionHeaders(requestHeaders, ENCRYPTION_KEY_PREFIX, req.targetOptions); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java index e8645b3eb384..ef0ffcd06cce 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java @@ -55,7 +55,8 @@ enum Option { VERSIONS("versions"), FIELDS("fields"), CUSTOMER_SUPPLIED_KEY("customerSuppliedKey"), - USER_PROJECT("userProject"); + USER_PROJECT("userProject"), + KMS_KEY_NAME("kmsKeyName"); private final String value; diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobInfoTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobInfoTest.java index 3160af0cc8da..460bdb1ea811 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobInfoTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobInfoTest.java @@ -69,8 +69,6 @@ public class BlobInfoTest { private static final String KMS_KEY_NAME = "projects/p/locations/kr-loc/keyRings/kr/cryptoKeys/key"; private static final StorageClass STORAGE_CLASS = StorageClass.COLDLINE; - // Unit tests include Customer Supplied encryption which may be confusing with the KMS KEY NAME - // as both can't be set at the same time. At least not right now. private static final BlobInfo BLOB_INFO = BlobInfo.newBuilder("b", "n", GENERATION) .setAcl(ACL) .setComponentCount(COMPONENT_COUNT) diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java index b49d16fbc340..644a887b93ec 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java @@ -91,6 +91,7 @@ public class BlobTest { private static final String KEY_SHA256 = "keySha"; private static final BlobInfo.CustomerEncryption CUSTOMER_ENCRYPTION = new BlobInfo.CustomerEncryption(ENCRYPTION_ALGORITHM, KEY_SHA256); + // TODO: Figure out CMEK + CSEK world private static final BlobInfo FULL_BLOB_INFO = BlobInfo.newBuilder("b", "n", GENERATION) .setAcl(ACLS) .setComponentCount(COMPONENT_COUNT) diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java index 34c14781857a..8f4097a48535 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java @@ -41,6 +41,7 @@ import com.google.common.collect.Lists; import com.google.common.io.BaseEncoding; +import com.sun.net.ssl.KeyManagerFactorySpi; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -97,10 +98,10 @@ public class BucketTest { .setNotFoundPage(NOT_FOUND_PAGE) .setLocation(LOCATION) .setStorageClass(STORAGE_CLASS) - .setDefaultKmsKeyName(DEFAULT_KMS_KEY_NAME) .setVersioningEnabled(VERSIONING_ENABLED) .setLabels(BUCKET_LABELS) .setRequesterPays(REQUESTER_PAYS) + .setDefaultKmsKeyName(DEFAULT_KMS_KEY_NAME) .build(); private static final BucketInfo BUCKET_INFO = BucketInfo.newBuilder("b").setMetageneration(42L).build(); @@ -134,12 +135,12 @@ private void initializeExpectedBucket(int optionsCalls) { replay(serviceMockReturnsOptions); expectedBucket = new Bucket(serviceMockReturnsOptions, new BucketInfo.BuilderImpl(BUCKET_INFO)); blobResults = ImmutableList.of( - new Blob(serviceMockReturnsOptions, - new BlobInfo.BuilderImpl(BlobInfo.newBuilder("b", "n1").build())), - new Blob(serviceMockReturnsOptions, - new BlobInfo.BuilderImpl(BlobInfo.newBuilder("b", "n2").build())), - new Blob(serviceMockReturnsOptions, - new BlobInfo.BuilderImpl(BlobInfo.newBuilder("b", "n3").build()))); + new Blob(serviceMockReturnsOptions, + new BlobInfo.BuilderImpl(BlobInfo.newBuilder("b", "n1").build())), + new Blob(serviceMockReturnsOptions, + new BlobInfo.BuilderImpl(BlobInfo.newBuilder("b", "n2").build())), + new Blob(serviceMockReturnsOptions, + new BlobInfo.BuilderImpl(BlobInfo.newBuilder("b", "n3").build()))); } private void initializeBucket() { @@ -677,6 +678,7 @@ public void testBuilder() { .setVersioningEnabled(VERSIONING_ENABLED) .setLabels(BUCKET_LABELS) .setRequesterPays(REQUESTER_PAYS) + .setDefaultKmsKeyName(DEFAULT_KMS_KEY_NAME) .build(); assertEquals("b", bucket.getName()); assertEquals(ACLS, bucket.getAcl()); @@ -696,6 +698,7 @@ public void testBuilder() { assertEquals(VERSIONING_ENABLED, bucket.versioningEnabled()); assertEquals(BUCKET_LABELS, bucket.getLabels()); assertEquals(REQUESTER_PAYS, bucket.requesterPays()); + assertEquals(DEFAULT_KMS_KEY_NAME, bucket.getDefaultKmsKeyName()); assertEquals(storage.getOptions(), bucket.getStorage().getOptions()); } } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java index dbab8bc4ad83..b2635726f382 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java @@ -561,6 +561,8 @@ public void testCreateBlobWithEncryptionKey() throws IOException { assertEquals(-1, byteStream.read(streamBytes)); } + // TODO: Add tests for KMS Key name + @Test public void testCreateBlobFromStream() throws IOException { Capture capturedStream = Capture.newInstance(); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index 3ebd0150e365..034bb6f8dc2e 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -27,11 +27,7 @@ import static org.junit.Assert.fail; import com.google.api.gax.paging.Page; -import com.google.cloud.Identity; -import com.google.cloud.Policy; -import com.google.cloud.ReadChannel; -import com.google.cloud.RestorableState; -import com.google.cloud.WriteChannel; +import com.google.cloud.*; import com.google.cloud.storage.Acl; import com.google.cloud.storage.Acl.Role; import com.google.cloud.storage.Acl.User; @@ -107,8 +103,8 @@ public class ITStorageTest { private static final byte[] COMPRESSED_CONTENT = BaseEncoding.base64() .decode("H4sIAAAAAAAAAPNIzcnJV3DPz0/PSVVwzskvTVEILskvSkxPVQQA/LySchsAAAA="); private static final Map BUCKET_LABELS = ImmutableMap.of("label1", "value1"); - private static final String SERVICE_ACCOUNT_EMAIL = "gcloud-devel@gs-project-accounts.iam.gserviceaccount.com"; - private static final String KMS_KEY_NAME = ""; // use real KMS KEY associated to the test project. + private static final String SERVICE_ACCOUNT_EMAIL = "spec-test-ruby-samples@gs-project-accounts.iam.gserviceaccount.com"; + private static final String KMS_KEY_NAME = "projects/spec-test-ruby-samples/locations/global/keyRings/a-key-ring-list-spec-test-ruby-samples/cryptoKeys/test"; // use real KMS KEY associated to the test project. @BeforeClass public static void beforeClass() throws NoSuchAlgorithmException, InvalidKeySpecException { @@ -185,14 +181,27 @@ public void testClearBucketDefaultKmsKeyName() throws ExecutionException, Interr try { assertEquals(KMS_KEY_NAME, remoteBucket.getDefaultKmsKeyName()); - remoteBucket = remoteBucket.toBuilder().setDefaultKmsKeyName(null).build(); - Bucket updatedBucket = storage.update(remoteBucket); - assertEquals(null, updatedBucket.getDefaultKmsKeyName()); + Bucket updatedBucket = remoteBucket.toBuilder().setDefaultKmsKeyName(null).build().update(); + assertNull(updatedBucket.getDefaultKmsKeyName()); } finally { RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); } } +// @Test +// public void testClearBucketDefaultKmsKeyNameTargetOptions() throws ExecutionException, InterruptedException { +// String bucketName = RemoteStorageHelper.generateBucketName(); +// Bucket remoteBucket = storage.create(BucketInfo.newBuilder(bucketName).setDefaultKmsKeyName(KMS_KEY_NAME).build()); +// +// try { +// assertEquals(KMS_KEY_NAME, remoteBucket.getDefaultKmsKeyName()); +// Bucket updatedBucket = remoteBucket.toBuilder().build().update(); +// assertNull(updatedBucket.getDefaultKmsKeyName()); +// } finally { +// RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); +// } +// } + @Test public void testCreateBlob() { String blobName = "test-create-blob"; @@ -228,7 +237,7 @@ public void testCreateBlobWithKmsKeyName() { assertNotNull(remoteBlob); assertEquals(blob.getBucket(), remoteBlob.getBucket()); assertEquals(blob.getName(), remoteBlob.getName()); - assertEquals(blob.getKmsKeyName(), remoteBlob.getKmsKeyName()); + assertTrue(remoteBlob.getKmsKeyName().startsWith(blob.getKmsKeyName())); byte[] readBytes = storage.readAllBytes(BUCKET, blobName); assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); } @@ -237,6 +246,7 @@ public void testCreateBlobWithKmsKeyName() { public void testCreateBlobWithDefaultKmsKeyName() throws ExecutionException, InterruptedException { String bucketName = RemoteStorageHelper.generateBucketName(); Bucket bucket = storage.create(BucketInfo.newBuilder(bucketName).setDefaultKmsKeyName(KMS_KEY_NAME).build()); + assertEquals(bucket.getDefaultKmsKeyName(), KMS_KEY_NAME); try { String blobName = "test-create-with-default-kms-key-name-blob"; @@ -245,8 +255,9 @@ public void testCreateBlobWithDefaultKmsKeyName() throws ExecutionException, Int assertNotNull(remoteBlob); assertEquals(blob.getBucket(), remoteBlob.getBucket()); assertEquals(blob.getName(), remoteBlob.getName()); - assertEquals(blob.getKmsKeyName(), remoteBlob.getKmsKeyName()); - byte[] readBytes = storage.readAllBytes(BUCKET, blobName); + assertNotNull(remoteBlob.getKmsKeyName()); + assertTrue(remoteBlob.getKmsKeyName().startsWith(KMS_KEY_NAME)); + byte[] readBytes = storage.readAllBytes(bucketName, blobName); assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); } finally { RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); @@ -455,7 +466,7 @@ public void testListBlobsKmsKeySelectedFields() throws InterruptedException { Blob remoteBlob = iterator.next(); assertEquals(BUCKET, remoteBlob.getBucket()); assertTrue(blobSet.contains(remoteBlob.getName())); - assertEquals(KMS_KEY_NAME, remoteBlob.getKmsKeyName()); + assertTrue(remoteBlob.getKmsKeyName().startsWith(KMS_KEY_NAME)); assertNull(remoteBlob.getContentType()); } } @@ -918,7 +929,8 @@ public void testRotateFromCustomerEncryptionToKmsKey() { assertEquals(BUCKET, copyWriter.getResult().getBucket()); assertEquals(targetBlobName, copyWriter.getResult().getName()); assertEquals(CONTENT_TYPE, copyWriter.getResult().getContentType()); - assertEquals(KMS_KEY_NAME, copyWriter.getResult().getKmsKeyName()); + assertNotNull(copyWriter.getResult().getKmsKeyName()); + assertTrue(copyWriter.getResult().getKmsKeyName().startsWith(KMS_KEY_NAME)); assertArrayEquals(BLOB_BYTE_CONTENT, copyWriter.getResult().getContent()); assertEquals(metadata, copyWriter.getResult().getMetadata()); assertTrue(copyWriter.isDone()); @@ -949,7 +961,7 @@ public void testCopyBlobUpdateMetadata() { } //Re-enable this test when it stops failing - //@Test + @Test public void testCopyBlobUpdateStorageClass() { String sourceBlobName = "test-copy-blob-update-storage-class-source"; BlobId source = BlobId.of(BUCKET, sourceBlobName); @@ -964,6 +976,7 @@ public void testCopyBlobUpdateStorageClass() { .newBuilder(BUCKET, targetBlobName).setStorageClass(StorageClass.COLDLINE).build(); Storage.CopyRequest req = Storage.CopyRequest.of(source, targetInfo); CopyWriter copyWriter = storage.copy(req); + Blob test = storage.get(source); assertEquals(BUCKET, copyWriter.getResult().getBucket()); assertEquals(targetBlobName, copyWriter.getResult().getName()); assertEquals(StorageClass.COLDLINE, copyWriter.getResult().getStorageClass()); @@ -1231,6 +1244,32 @@ public void testReadAndWriteChannelWithEncryptionKey() throws IOException { assertTrue(storage.delete(BUCKET, blobName)); } + @Test + public void testReadAndWriteChannelWithKmsKey() throws IOException { + String blobName = "test-read-write-channel-with-customer-kms-key-blob"; + BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); + byte[] stringBytes; + try (WriteChannel writer = storage.writer(blob, + Storage.BlobWriteOption.kmsKeyName(KMS_KEY_NAME))) { + stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8); + writer.write(ByteBuffer.wrap(BLOB_BYTE_CONTENT)); + writer.write(ByteBuffer.wrap(stringBytes)); + } + ByteBuffer readBytes; + ByteBuffer readStringBytes; + try (ReadChannel reader = + storage.reader(blob.getBlobId())) { + readBytes = ByteBuffer.allocate(BLOB_BYTE_CONTENT.length); + readStringBytes = ByteBuffer.allocate(stringBytes.length); + reader.read(readBytes); + reader.read(readStringBytes); + } + assertArrayEquals(BLOB_BYTE_CONTENT, readBytes.array()); + assertEquals(BLOB_STRING_CONTENT, new String(readStringBytes.array(), UTF_8)); + assertTrue(storage.get(blob.getBlobId()).getKmsKeyName().startsWith(KMS_KEY_NAME)); + assertTrue(storage.delete(BUCKET, blobName)); + } + @Test public void testReadAndWriteChannelsWithDifferentFileSize() throws IOException { String blobNamePrefix = "test-read-and-write-channels-blob-"; From 8d042d2544d4d22ebd8d4564074a68a39722d698 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Mon, 7 May 2018 00:16:35 -0700 Subject: [PATCH 05/13] Clean up and add tests --- .../java/com/google/cloud/storage/Bucket.java | 19 --- .../com/google/cloud/storage/Storage.java | 24 +--- .../com/google/cloud/storage/StorageImpl.java | 2 - .../cloud/storage/spi/v1/HttpStorageRpc.java | 10 +- .../com/google/cloud/storage/BlobTest.java | 1 - .../com/google/cloud/storage/BucketTest.java | 13 +- .../google/cloud/storage/StorageImplTest.java | 2 - .../cloud/storage/it/ITStorageTest.java | 133 ++++++++++-------- 8 files changed, 86 insertions(+), 118 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java index c5a399391de9..07254e1185f6 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java @@ -190,9 +190,6 @@ private Tuple toTargetOption(BlobInfo blobIn case USER_PROJECT: return Tuple.of(blobInfo, Storage.BlobTargetOption.userProject((String) getValue())); - case KMS_KEY_NAME: - return Tuple.of(blobInfo, - Storage.BlobTargetOption.kmsKeyName((String) getValue())); default: throw new AssertionError("Unexpected enum value"); } @@ -277,13 +274,6 @@ public static BlobTargetOption userProject(String userProject) { return new BlobTargetOption(StorageRpc.Option.USER_PROJECT, userProject); } - /** - * Returns an option for blob's kms key name. - */ - public static BlobTargetOption kmsKeyName(String kmsKeyName) { - return new BlobTargetOption(StorageRpc.Option.KMS_KEY_NAME, kmsKeyName); - } - static Tuple toTargetOptions( BlobInfo info, BlobTargetOption... options) { Set optionSet = @@ -355,8 +345,6 @@ private Tuple toWriteOption(BlobInfo blobInfo Storage.BlobWriteOption.encryptionKey((String) value)); case USER_PROJECT: return Tuple.of(blobInfo, Storage.BlobWriteOption.userProject((String) value)); - case KMS_KEY_NAME: - return Tuple.of(blobInfo, Storage.BlobWriteOption.kmsKeyName((String) value)); default: throw new AssertionError("Unexpected enum value"); } @@ -482,13 +470,6 @@ public static BlobWriteOption userProject(String userProject) { return new BlobWriteOption(Storage.BlobWriteOption.Option.USER_PROJECT, userProject); } - /** - * Returns an option for blob's kms key name. - */ - public static BlobWriteOption kmsKeyName(String kmsKeyName) { - return new BlobWriteOption(Storage.BlobWriteOption.Option.KMS_KEY_NAME, kmsKeyName); - } - static Tuple toWriteOptions( BlobInfo info, BlobWriteOption... options) { Set optionSet = diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 8d6e85c06204..1249493a71e9 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -208,15 +208,6 @@ public static BucketTargetOption metagenerationNotMatch() { public static BucketTargetOption userProject(String userProject) { return new BucketTargetOption(StorageRpc.Option.USER_PROJECT, userProject); } - -// /** -// * Returns an option to define the defaultKmsKeyName. -// */ -// public static BucketTargetOption defaultKmsKeyName(String defaultKmsKeyName) { -// defaultKmsKeyName = defaultKmsKeyName != null -// ? new String(defaultKmsKeyName) : Data.nullOf(String.class); -// return new BucketTargetOption(StorageRpc.Option.KMS_KEY_NAME, defaultKmsKeyName); -// } } /** @@ -394,15 +385,6 @@ public static BlobTargetOption encryptionKey(String key) { return new BlobTargetOption(StorageRpc.Option.CUSTOMER_SUPPLIED_KEY, key); } - /** - * Returns an option to set a KMS key resource name of the blob. - * - * @param kmsKeyName the KMS crypto key resource name - */ - public static BlobTargetOption kmsKeyName(String kmsKeyName) { - return new BlobTargetOption(StorageRpc.Option.KMS_KEY_NAME, kmsKeyName); - } - static Tuple convert(BlobInfo info, BlobWriteOption... options) { BlobInfo.Builder infoBuilder = info.toBuilder().setCrc32c(null).setMd5(null); List targetOptions = Lists.newArrayListWithCapacity(options.length); @@ -436,7 +418,7 @@ class BlobWriteOption implements Serializable { enum Option { PREDEFINED_ACL, IF_GENERATION_MATCH, IF_GENERATION_NOT_MATCH, IF_METAGENERATION_MATCH, - IF_METAGENERATION_NOT_MATCH, IF_MD5_MATCH, IF_CRC32C_MATCH, CUSTOMER_SUPPLIED_KEY, USER_PROJECT, KMS_KEY_NAME; + IF_METAGENERATION_NOT_MATCH, IF_MD5_MATCH, IF_CRC32C_MATCH, CUSTOMER_SUPPLIED_KEY, USER_PROJECT; StorageRpc.Option toRpcOption() { return StorageRpc.Option.valueOf(this.name()); @@ -561,10 +543,6 @@ public static BlobWriteOption encryptionKey(String key) { public static BlobWriteOption userProject(String userProject) { return new BlobWriteOption(Option.USER_PROJECT, userProject); } - - public static BlobWriteOption kmsKeyName(String kmsKeyName) { - return new BlobWriteOption(Option.KMS_KEY_NAME, kmsKeyName); - } } /** diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index 20ebab1f2174..e49445d13646 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -437,11 +437,9 @@ public CopyWriter copy(final CopyRequest copyRequest) { RewriteResponse rewriteResponse = runWithRetries(new Callable() { @Override public RewriteResponse call() { - return storageRpc.openRewrite(new StorageRpc.RewriteRequest(source, sourceOptions, copyRequest.overrideInfo(), targetObject, targetOptions, copyRequest.getMegabytesCopiedPerChunk())); - } }, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock()); return new CopyWriter(getOptions(), rewriteResponse); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index 19b03d113a5f..e88f43bdf31f 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -81,6 +81,8 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; + +import jdk.nashorn.internal.runtime.regexp.joni.exception.ValueException; import org.apache.http.HttpStatus; public class HttpStorageRpc implements StorageRpc { @@ -426,7 +428,6 @@ public StorageObject get(StorageObject object, Map options) { public Bucket patch(Bucket bucket, Map options) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_PATCH_BUCKET); Scope scope = tracer.withSpan(span); - try { return storage.buckets() .patch(bucket.getName(), bucket) @@ -790,11 +791,7 @@ private RewriteResponse rewrite(RewriteRequest req, String token) { if (userProject == null) { userProject = Option.USER_PROJECT.getString(req.targetOptions); } - - String kmsKeyName = Option.KMS_KEY_NAME.getString(req.targetOptions); - if (kmsKeyName == null) { - kmsKeyName = req.target.getKmsKeyName(); - } + String kmsKeyName = req.target.getKmsKeyName(); Long maxBytesRewrittenPerCall = req.megabytesRewrittenPerCall != null ? req.megabytesRewrittenPerCall * MEGABYTE : null; @@ -816,7 +813,6 @@ private RewriteResponse rewrite(RewriteRequest req, String token) { .setIfGenerationNotMatch(Option.IF_GENERATION_NOT_MATCH.getLong(req.targetOptions)) .setUserProject(userProject) .setDestinationKmsKeyName(kmsKeyName); - HttpHeaders requestHeaders = rewrite.getRequestHeaders(); setEncryptionHeaders(requestHeaders, SOURCE_ENCRYPTION_KEY_PREFIX, req.sourceOptions); setEncryptionHeaders(requestHeaders, ENCRYPTION_KEY_PREFIX, req.targetOptions); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java index 644a887b93ec..b49d16fbc340 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java @@ -91,7 +91,6 @@ public class BlobTest { private static final String KEY_SHA256 = "keySha"; private static final BlobInfo.CustomerEncryption CUSTOMER_ENCRYPTION = new BlobInfo.CustomerEncryption(ENCRYPTION_ALGORITHM, KEY_SHA256); - // TODO: Figure out CMEK + CSEK world private static final BlobInfo FULL_BLOB_INFO = BlobInfo.newBuilder("b", "n", GENERATION) .setAcl(ACLS) .setComponentCount(COMPONENT_COUNT) diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java index 8f4097a48535..94c8ad715a0a 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java @@ -41,7 +41,6 @@ import com.google.common.collect.Lists; import com.google.common.io.BaseEncoding; -import com.sun.net.ssl.KeyManagerFactorySpi; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -135,12 +134,12 @@ private void initializeExpectedBucket(int optionsCalls) { replay(serviceMockReturnsOptions); expectedBucket = new Bucket(serviceMockReturnsOptions, new BucketInfo.BuilderImpl(BUCKET_INFO)); blobResults = ImmutableList.of( - new Blob(serviceMockReturnsOptions, - new BlobInfo.BuilderImpl(BlobInfo.newBuilder("b", "n1").build())), - new Blob(serviceMockReturnsOptions, - new BlobInfo.BuilderImpl(BlobInfo.newBuilder("b", "n2").build())), - new Blob(serviceMockReturnsOptions, - new BlobInfo.BuilderImpl(BlobInfo.newBuilder("b", "n3").build()))); + new Blob(serviceMockReturnsOptions, + new BlobInfo.BuilderImpl(BlobInfo.newBuilder("b", "n1").build())), + new Blob(serviceMockReturnsOptions, + new BlobInfo.BuilderImpl(BlobInfo.newBuilder("b", "n2").build())), + new Blob(serviceMockReturnsOptions, + new BlobInfo.BuilderImpl(BlobInfo.newBuilder("b", "n3").build()))); } private void initializeBucket() { diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java index b2635726f382..dbab8bc4ad83 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java @@ -561,8 +561,6 @@ public void testCreateBlobWithEncryptionKey() throws IOException { assertEquals(-1, byteStream.read(streamBytes)); } - // TODO: Add tests for KMS Key name - @Test public void testCreateBlobFromStream() throws IOException { Capture capturedStream = Capture.newInstance(); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index 034bb6f8dc2e..64efc81f1f05 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -27,7 +27,11 @@ import static org.junit.Assert.fail; import com.google.api.gax.paging.Page; -import com.google.cloud.*; +import com.google.cloud.Identity; +import com.google.cloud.Policy; +import com.google.cloud.ReadChannel; +import com.google.cloud.RestorableState; +import com.google.cloud.WriteChannel; import com.google.cloud.storage.Acl; import com.google.cloud.storage.Acl.Role; import com.google.cloud.storage.Acl.User; @@ -103,8 +107,9 @@ public class ITStorageTest { private static final byte[] COMPRESSED_CONTENT = BaseEncoding.base64() .decode("H4sIAAAAAAAAAPNIzcnJV3DPz0/PSVVwzskvTVEILskvSkxPVQQA/LySchsAAAA="); private static final Map BUCKET_LABELS = ImmutableMap.of("label1", "value1"); - private static final String SERVICE_ACCOUNT_EMAIL = "spec-test-ruby-samples@gs-project-accounts.iam.gserviceaccount.com"; - private static final String KMS_KEY_NAME = "projects/spec-test-ruby-samples/locations/global/keyRings/a-key-ring-list-spec-test-ruby-samples/cryptoKeys/test"; // use real KMS KEY associated to the test project. + private static final String SERVICE_ACCOUNT_EMAIL = "gcloud-devel@gs-project-accounts.iam.gserviceaccount.com"; + private static final String KMS_KEY_NAME_1 = "projects/gcloud-devel/locations/us/keyRings/gcs_kms_key_ring_us/cryptoKeys/key"; + private static final String KMS_KEY_NAME_2 = "projects/gcloud-devel/locations/us/keyRings/gcs_kms_key_ring_us/cryptoKeys/key2"; @BeforeClass public static void beforeClass() throws NoSuchAlgorithmException, InvalidKeySpecException { @@ -177,10 +182,11 @@ public void testGetBucketEmptyFields() { @Test public void testClearBucketDefaultKmsKeyName() throws ExecutionException, InterruptedException { String bucketName = RemoteStorageHelper.generateBucketName(); - Bucket remoteBucket = storage.create(BucketInfo.newBuilder(bucketName).setDefaultKmsKeyName(KMS_KEY_NAME).build()); + Bucket remoteBucket = storage.create(BucketInfo.newBuilder(bucketName) + .setDefaultKmsKeyName(KMS_KEY_NAME_1).setLocation("US").build()); try { - assertEquals(KMS_KEY_NAME, remoteBucket.getDefaultKmsKeyName()); + assertEquals(KMS_KEY_NAME_1, remoteBucket.getDefaultKmsKeyName()); Bucket updatedBucket = remoteBucket.toBuilder().setDefaultKmsKeyName(null).build().update(); assertNull(updatedBucket.getDefaultKmsKeyName()); } finally { @@ -188,19 +194,20 @@ public void testClearBucketDefaultKmsKeyName() throws ExecutionException, Interr } } -// @Test -// public void testClearBucketDefaultKmsKeyNameTargetOptions() throws ExecutionException, InterruptedException { -// String bucketName = RemoteStorageHelper.generateBucketName(); -// Bucket remoteBucket = storage.create(BucketInfo.newBuilder(bucketName).setDefaultKmsKeyName(KMS_KEY_NAME).build()); -// -// try { -// assertEquals(KMS_KEY_NAME, remoteBucket.getDefaultKmsKeyName()); -// Bucket updatedBucket = remoteBucket.toBuilder().build().update(); -// assertNull(updatedBucket.getDefaultKmsKeyName()); -// } finally { -// RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); -// } -// } + @Test + public void testUpdateBucketDefaultKmsKeyName() throws ExecutionException, InterruptedException { + String bucketName = RemoteStorageHelper.generateBucketName(); + Bucket remoteBucket = storage.create(BucketInfo.newBuilder(bucketName) + .setDefaultKmsKeyName(KMS_KEY_NAME_1).setLocation("US").build()); + + try { + assertEquals(KMS_KEY_NAME_1, remoteBucket.getDefaultKmsKeyName()); + Bucket updatedBucket = remoteBucket.toBuilder().setDefaultKmsKeyName(KMS_KEY_NAME_2).build().update(); + assertEquals(KMS_KEY_NAME_2, updatedBucket.getDefaultKmsKeyName()); + } finally { + RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); + } + } @Test public void testCreateBlob() { @@ -232,7 +239,7 @@ public void testCreateBlobWithEncryptionKey() { @Test public void testCreateBlobWithKmsKeyName() { String blobName = "test-create-with-kms-key-name-blob"; - BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).setKmsKeyName(KMS_KEY_NAME).build(); + BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).setKmsKeyName(KMS_KEY_NAME_1).build(); Blob remoteBlob = storage.create(blob, BLOB_BYTE_CONTENT); assertNotNull(remoteBlob); assertEquals(blob.getBucket(), remoteBlob.getBucket()); @@ -242,11 +249,24 @@ public void testCreateBlobWithKmsKeyName() { assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); } + @Test + public void testCreateBlobWithKmsKeyNameAndCustomerSuppliedKey() { + try { + String blobName = "test-create-with-kms-key-name-blob"; + BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).setKmsKeyName(KMS_KEY_NAME_1).build(); + storage.create(blob, BLOB_BYTE_CONTENT, Storage.BlobTargetOption.encryptionKey(KEY)); + fail("StorageException was expected"); // can't supply both. + } catch (StorageException ex) { + // expected + } + } + @Test public void testCreateBlobWithDefaultKmsKeyName() throws ExecutionException, InterruptedException { String bucketName = RemoteStorageHelper.generateBucketName(); - Bucket bucket = storage.create(BucketInfo.newBuilder(bucketName).setDefaultKmsKeyName(KMS_KEY_NAME).build()); - assertEquals(bucket.getDefaultKmsKeyName(), KMS_KEY_NAME); + Bucket bucket = storage.create(BucketInfo.newBuilder(bucketName) + .setDefaultKmsKeyName(KMS_KEY_NAME_1).setLocation("US").build()); + assertEquals(bucket.getDefaultKmsKeyName(), KMS_KEY_NAME_1); try { String blobName = "test-create-with-default-kms-key-name-blob"; @@ -256,7 +276,7 @@ public void testCreateBlobWithDefaultKmsKeyName() throws ExecutionException, Int assertEquals(blob.getBucket(), remoteBlob.getBucket()); assertEquals(blob.getName(), remoteBlob.getName()); assertNotNull(remoteBlob.getKmsKeyName()); - assertTrue(remoteBlob.getKmsKeyName().startsWith(KMS_KEY_NAME)); + assertTrue(remoteBlob.getKmsKeyName().startsWith(KMS_KEY_NAME_1)); byte[] readBytes = storage.readAllBytes(bucketName, blobName); assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); } finally { @@ -439,11 +459,11 @@ public void testListBlobsKmsKeySelectedFields() throws InterruptedException { "test-list-blobs-selected-field-kms-key-name-blob2"}; BlobInfo blob1 = BlobInfo.newBuilder(BUCKET, blobNames[0]) .setContentType(CONTENT_TYPE) - .setKmsKeyName(KMS_KEY_NAME) + .setKmsKeyName(KMS_KEY_NAME_1) .build(); BlobInfo blob2 = BlobInfo.newBuilder(BUCKET, blobNames[1]) .setContentType(CONTENT_TYPE) - .setKmsKeyName(KMS_KEY_NAME) + .setKmsKeyName(KMS_KEY_NAME_1) .build(); Blob remoteBlob1 = storage.create(blob1); Blob remoteBlob2 = storage.create(blob2); @@ -466,7 +486,7 @@ public void testListBlobsKmsKeySelectedFields() throws InterruptedException { Blob remoteBlob = iterator.next(); assertEquals(BUCKET, remoteBlob.getBucket()); assertTrue(blobSet.contains(remoteBlob.getName())); - assertTrue(remoteBlob.getKmsKeyName().startsWith(KMS_KEY_NAME)); + assertTrue(remoteBlob.getKmsKeyName().startsWith(KMS_KEY_NAME_1)); assertNull(remoteBlob.getContentType()); } } @@ -550,8 +570,6 @@ public void testListBlobRequesterPays() throws InterruptedException { } } - - @Test(timeout = 15000) public void testListBlobsVersioned() throws ExecutionException, InterruptedException { String bucketName = RemoteStorageHelper.generateBucketName(); @@ -918,7 +936,7 @@ public void testRotateFromCustomerEncryptionToKmsKey() { BlobInfo target = BlobInfo.newBuilder(BUCKET, targetBlobName) .setContentType(CONTENT_TYPE) .setMetadata(metadata) - .setKmsKeyName(KMS_KEY_NAME) + .setKmsKeyName(KMS_KEY_NAME_1) .build(); Storage.CopyRequest req = Storage.CopyRequest.newBuilder() .setSource(source) @@ -930,13 +948,41 @@ public void testRotateFromCustomerEncryptionToKmsKey() { assertEquals(targetBlobName, copyWriter.getResult().getName()); assertEquals(CONTENT_TYPE, copyWriter.getResult().getContentType()); assertNotNull(copyWriter.getResult().getKmsKeyName()); - assertTrue(copyWriter.getResult().getKmsKeyName().startsWith(KMS_KEY_NAME)); + assertTrue(copyWriter.getResult().getKmsKeyName().startsWith(KMS_KEY_NAME_1)); assertArrayEquals(BLOB_BYTE_CONTENT, copyWriter.getResult().getContent()); assertEquals(metadata, copyWriter.getResult().getMetadata()); assertTrue(copyWriter.isDone()); assertTrue(storage.delete(BUCKET, targetBlobName)); } + @Test + public void testRotateFromCustomerEncryptionToKmsKeyWithCustomerEncrytion() { + String sourceBlobName = "test-copy-blob-encryption-key-source"; + BlobId source = BlobId.of(BUCKET, sourceBlobName); + ImmutableMap metadata = ImmutableMap.of("k", "v"); + Blob remoteBlob = storage.create(BlobInfo.newBuilder(source).build(), BLOB_BYTE_CONTENT, + Storage.BlobTargetOption.encryptionKey(KEY)); + assertNotNull(remoteBlob); + String targetBlobName = "test-copy-blob-kms-key-target"; + BlobInfo target = BlobInfo.newBuilder(BUCKET, targetBlobName) + .setContentType(CONTENT_TYPE) + .setMetadata(metadata) + .setKmsKeyName(KMS_KEY_NAME_1) + .build(); + try { + Storage.CopyRequest req = Storage.CopyRequest.newBuilder() + .setSource(source) + .setSourceOptions(Storage.BlobSourceOption.decryptionKey(BASE64_KEY)) + .setTarget(target, Storage.BlobTargetOption.encryptionKey(KEY)) + .build(); + storage.copy(req); + fail("StorageException was expected"); + } catch (StorageException ex) { + // expected + } + } + + @Test public void testCopyBlobUpdateMetadata() { String sourceBlobName = "test-copy-blob-update-metadata-source"; @@ -961,7 +1007,7 @@ public void testCopyBlobUpdateMetadata() { } //Re-enable this test when it stops failing - @Test + //@Test public void testCopyBlobUpdateStorageClass() { String sourceBlobName = "test-copy-blob-update-storage-class-source"; BlobId source = BlobId.of(BUCKET, sourceBlobName); @@ -976,7 +1022,6 @@ public void testCopyBlobUpdateStorageClass() { .newBuilder(BUCKET, targetBlobName).setStorageClass(StorageClass.COLDLINE).build(); Storage.CopyRequest req = Storage.CopyRequest.of(source, targetInfo); CopyWriter copyWriter = storage.copy(req); - Blob test = storage.get(source); assertEquals(BUCKET, copyWriter.getResult().getBucket()); assertEquals(targetBlobName, copyWriter.getResult().getName()); assertEquals(StorageClass.COLDLINE, copyWriter.getResult().getStorageClass()); @@ -1244,32 +1289,6 @@ public void testReadAndWriteChannelWithEncryptionKey() throws IOException { assertTrue(storage.delete(BUCKET, blobName)); } - @Test - public void testReadAndWriteChannelWithKmsKey() throws IOException { - String blobName = "test-read-write-channel-with-customer-kms-key-blob"; - BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); - byte[] stringBytes; - try (WriteChannel writer = storage.writer(blob, - Storage.BlobWriteOption.kmsKeyName(KMS_KEY_NAME))) { - stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8); - writer.write(ByteBuffer.wrap(BLOB_BYTE_CONTENT)); - writer.write(ByteBuffer.wrap(stringBytes)); - } - ByteBuffer readBytes; - ByteBuffer readStringBytes; - try (ReadChannel reader = - storage.reader(blob.getBlobId())) { - readBytes = ByteBuffer.allocate(BLOB_BYTE_CONTENT.length); - readStringBytes = ByteBuffer.allocate(stringBytes.length); - reader.read(readBytes); - reader.read(readStringBytes); - } - assertArrayEquals(BLOB_BYTE_CONTENT, readBytes.array()); - assertEquals(BLOB_STRING_CONTENT, new String(readStringBytes.array(), UTF_8)); - assertTrue(storage.get(blob.getBlobId()).getKmsKeyName().startsWith(KMS_KEY_NAME)); - assertTrue(storage.delete(BUCKET, blobName)); - } - @Test public void testReadAndWriteChannelsWithDifferentFileSize() throws IOException { String blobNamePrefix = "test-read-and-write-channels-blob-"; From 116220c0220e49220883975c20f809acde68fcf9 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Mon, 7 May 2018 10:28:27 -0700 Subject: [PATCH 06/13] Address comments --- google-cloud-storage/pom.xml | 1 - .../src/main/java/com/google/cloud/storage/Blob.java | 1 + .../src/main/java/com/google/cloud/storage/BlobInfo.java | 4 ++++ .../src/main/java/com/google/cloud/storage/Bucket.java | 1 + .../src/main/java/com/google/cloud/storage/BucketInfo.java | 4 ++++ .../java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java | 1 - 6 files changed, 10 insertions(+), 2 deletions(-) diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index d7765f318844..c88f3ba7ba30 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -29,7 +29,6 @@ com.google.apis google-api-services-storage - v1-rev125-1.23.0 compile diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java index 0f6a8e379485..e7ffbc7515d6 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java @@ -402,6 +402,7 @@ Builder setCustomerEncryption(CustomerEncryption customerEncryption) { return this; } + @GcpLaunchStage.Beta @Override public Builder setKmsKeyName(String kmsKeyName) { infoBuilder.setKmsKeyName(kmsKeyName); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java index 71e711ee95c7..17e92fa96586 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java @@ -24,6 +24,7 @@ import com.google.api.services.storage.model.ObjectAccessControl; import com.google.api.services.storage.model.StorageObject; import com.google.api.services.storage.model.StorageObject.Owner; +import com.google.cloud.GcpLaunchStage; import com.google.cloud.storage.Blob.Builder; import com.google.common.base.Function; import com.google.common.base.MoreObjects; @@ -271,6 +272,7 @@ public abstract static class Builder { * * Sets the blob's kmsKeyName. */ + @GcpLaunchStage.Beta public abstract Builder setKmsKeyName(String kmsKeyName); /** @@ -484,6 +486,7 @@ Builder setCustomerEncryption(CustomerEncryption customerEncryption) { return this; } + @GcpLaunchStage.Beta @Override public Builder setKmsKeyName(String kmsKeyName) { this.kmsKeyName = kmsKeyName; @@ -756,6 +759,7 @@ public StorageClass getStorageClass() { /** * Returns the Cloud KMS key used to encrypt the blob, if any. */ + @GcpLaunchStage.Beta public String getKmsKeyName() { return kmsKeyName; } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java index 07254e1185f6..046977707d34 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java @@ -612,6 +612,7 @@ public Builder setLabels(Map labels) { return this; } + @GcpLaunchStage.Beta @Override public Builder setDefaultKmsKeyName(String defaultKmsKeyName) { infoBuilder.setDefaultKmsKeyName(defaultKmsKeyName); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java index 8a0a24ef610e..e2fe369aa375 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java @@ -31,6 +31,7 @@ import com.google.api.services.storage.model.Bucket.Owner; import com.google.api.services.storage.model.Bucket.Versioning; import com.google.api.services.storage.model.Bucket.Website; +import com.google.cloud.GcpLaunchStage; import com.google.cloud.storage.Acl.Entity; import com.google.common.base.Function; import com.google.common.base.MoreObjects; @@ -428,6 +429,7 @@ public abstract static class Builder { /** * Sets the default Cloud KMS key name for this bucket. */ + @GcpLaunchStage.Beta public abstract Builder setDefaultKmsKeyName(String defaultKmsKeyName); /** @@ -593,6 +595,7 @@ public Builder setLabels(Map labels) { return this; } + @GcpLaunchStage.Beta @Override public Builder setDefaultKmsKeyName(String defaultKmsKeyName) { this.defaultKmsKeyName = defaultKmsKeyName != null @@ -782,6 +785,7 @@ public Map getLabels() { /** * Returns the default Cloud KMS key to be applied to newly inserted objects in this bucket. */ + @GcpLaunchStage.Beta public String getDefaultKmsKeyName() { return defaultKmsKeyName; } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index e88f43bdf31f..d9717d3c8ac9 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -82,7 +82,6 @@ import java.util.List; import java.util.Map; -import jdk.nashorn.internal.runtime.regexp.joni.exception.ValueException; import org.apache.http.HttpStatus; public class HttpStorageRpc implements StorageRpc { From afd6eb39cb087a1b86e57bd0439396a3808a5ccb Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Mon, 7 May 2018 11:04:11 -0700 Subject: [PATCH 07/13] Update storage apiary client library version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 53d7a154231e..58edfcabf648 100644 --- a/pom.xml +++ b/pom.xml @@ -196,7 +196,7 @@ com.google.apis google-api-services-storage - v1-rev114-1.23.0 + v1-rev125-1.23.0 com.google.apis From fa6b526dea6437aaf463cf9876823d494ac867d1 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Tue, 8 May 2018 08:08:22 -0700 Subject: [PATCH 08/13] Use BlobTargetOption and BlobWriteOption --- .../com/google/cloud/storage/BlobInfo.java | 2 +- .../java/com/google/cloud/storage/Bucket.java | 16 +++++ .../com/google/cloud/storage/Storage.java | 31 ++++++--- .../cloud/storage/spi/v1/HttpStorageRpc.java | 3 +- .../com/google/cloud/storage/BlobTest.java | 4 ++ .../google/cloud/storage/StorageImplTest.java | 5 ++ .../cloud/storage/it/ITStorageTest.java | 63 ++++++++++++++----- 7 files changed, 98 insertions(+), 26 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java index 17e92fa96586..c31b61edfe5c 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java @@ -273,7 +273,7 @@ public abstract static class Builder { * Sets the blob's kmsKeyName. */ @GcpLaunchStage.Beta - public abstract Builder setKmsKeyName(String kmsKeyName); + abstract Builder setKmsKeyName(String kmsKeyName); /** * Creates a {@code BlobInfo} object. diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java index 046977707d34..179e8a5cbfef 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java @@ -187,6 +187,9 @@ private Tuple toTargetOption(BlobInfo blobIn case CUSTOMER_SUPPLIED_KEY: return Tuple.of(blobInfo, Storage.BlobTargetOption.encryptionKey((String) getValue())); + case KMS_KEY_NAME: + return Tuple.of(blobInfo, + Storage.BlobTargetOption.kmsKeyName((String) getValue())); case USER_PROJECT: return Tuple.of(blobInfo, Storage.BlobTargetOption.userProject((String) getValue())); @@ -266,6 +269,16 @@ public static BlobTargetOption encryptionKey(String key) { return new BlobTargetOption(StorageRpc.Option.CUSTOMER_SUPPLIED_KEY, key); } + /** + * Returns an option to set a customer-managed KMS key for server-side encryption of the + * blob. + * + * @param kmsKeyName the KMS key resource id + */ + public static BlobTargetOption kmsKeyName(String kmsKeyName) { + return new BlobTargetOption(StorageRpc.Option.KMS_KEY_NAME, kmsKeyName); + } + /** * Returns an option for blob's billing user project. This option is only used by the buckets with * 'requester_pays' flag. @@ -343,6 +356,9 @@ private Tuple toWriteOption(BlobInfo blobInfo case CUSTOMER_SUPPLIED_KEY: return Tuple.of(blobInfo, Storage.BlobWriteOption.encryptionKey((String) value)); + case KMS_KEY_NAME: + return Tuple.of(blobInfo, + Storage.BlobWriteOption.kmsKeyName((String) value)); case USER_PROJECT: return Tuple.of(blobInfo, Storage.BlobWriteOption.userProject((String) value)); default: diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 1249493a71e9..194b10fea07e 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -19,19 +19,11 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; -import com.google.api.client.util.Data; -import com.google.api.core.BetaApi; import com.google.api.gax.paging.Page; import com.google.auth.ServiceAccountSigner; import com.google.auth.ServiceAccountSigner.SigningException; -import com.google.cloud.FieldSelector; +import com.google.cloud.*; import com.google.cloud.FieldSelector.Helper; -import com.google.cloud.GcpLaunchStage; -import com.google.cloud.Policy; -import com.google.cloud.ReadChannel; -import com.google.cloud.Service; -import com.google.cloud.Tuple; -import com.google.cloud.WriteChannel; import com.google.cloud.storage.Acl.Entity; import com.google.cloud.storage.spi.v1.StorageRpc; import com.google.common.collect.ImmutableList; @@ -385,6 +377,14 @@ public static BlobTargetOption encryptionKey(String key) { return new BlobTargetOption(StorageRpc.Option.CUSTOMER_SUPPLIED_KEY, key); } + /** + * Returns an option to set a customer-managed key for server-side encryption of the blob. + */ + @GcpLaunchStage.Beta + public static BlobTargetOption kmsKeyName(String kmsKeyName) { + return new BlobTargetOption(StorageRpc.Option.KMS_KEY_NAME, kmsKeyName); + } + static Tuple convert(BlobInfo info, BlobWriteOption... options) { BlobInfo.Builder infoBuilder = info.toBuilder().setCrc32c(null).setMd5(null); List targetOptions = Lists.newArrayListWithCapacity(options.length); @@ -418,7 +418,7 @@ class BlobWriteOption implements Serializable { enum Option { PREDEFINED_ACL, IF_GENERATION_MATCH, IF_GENERATION_NOT_MATCH, IF_METAGENERATION_MATCH, - IF_METAGENERATION_NOT_MATCH, IF_MD5_MATCH, IF_CRC32C_MATCH, CUSTOMER_SUPPLIED_KEY, USER_PROJECT; + IF_METAGENERATION_NOT_MATCH, IF_MD5_MATCH, IF_CRC32C_MATCH, CUSTOMER_SUPPLIED_KEY, KMS_KEY_NAME, USER_PROJECT; StorageRpc.Option toRpcOption() { return StorageRpc.Option.valueOf(this.name()); @@ -536,6 +536,17 @@ public static BlobWriteOption encryptionKey(String key) { return new BlobWriteOption(Option.CUSTOMER_SUPPLIED_KEY, key); } + /** + * Returns an option to set a customer-managed KMS key for server-side encryption of the + * blob. + * + * @param kmsKeyName the KMS key resource id + */ + @GcpLaunchStage.Beta + public static BlobWriteOption kmsKeyName(String kmsKeyName) { + return new BlobWriteOption(Option.KMS_KEY_NAME, kmsKeyName); + } + /** * Returns an option for blob's billing user project. This option is only used by the buckets with * 'requester_pays' flag. diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index d9717d3c8ac9..68a2e888d29e 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -286,6 +286,7 @@ public StorageObject create(StorageObject storageObject, final InputStream conte .setIfGenerationMatch(Option.IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(Option.IF_GENERATION_NOT_MATCH.getLong(options)) .setUserProject(Option.USER_PROJECT.getString(options)) + .setKmsKeyName(Option.KMS_KEY_NAME.getString(options)) .execute(); } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); @@ -790,7 +791,7 @@ private RewriteResponse rewrite(RewriteRequest req, String token) { if (userProject == null) { userProject = Option.USER_PROJECT.getString(req.targetOptions); } - String kmsKeyName = req.target.getKmsKeyName(); + String kmsKeyName = Option.KMS_KEY_NAME.getString(req.targetOptions); Long maxBytesRewrittenPerCall = req.megabytesRewrittenPerCall != null ? req.megabytesRewrittenPerCall * MEGABYTE : null; diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java index b49d16fbc340..a15544f68959 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java @@ -91,6 +91,7 @@ public class BlobTest { private static final String KEY_SHA256 = "keySha"; private static final BlobInfo.CustomerEncryption CUSTOMER_ENCRYPTION = new BlobInfo.CustomerEncryption(ENCRYPTION_ALGORITHM, KEY_SHA256); + private static final String KMS_KEY_NAME = "projects/p/locations/kr-loc/keyRings/kr/cryptoKeys/key"; private static final BlobInfo FULL_BLOB_INFO = BlobInfo.newBuilder("b", "n", GENERATION) .setAcl(ACLS) .setComponentCount(COMPONENT_COUNT) @@ -113,6 +114,7 @@ public class BlobTest { .setUpdateTime(UPDATE_TIME) .setCreateTime(CREATE_TIME) .setCustomerEncryption(CUSTOMER_ENCRYPTION) + .setKmsKeyName(KMS_KEY_NAME) .build(); private static final BlobInfo BLOB_INFO = BlobInfo.newBuilder("b", "n") .setMetageneration(42L) @@ -457,6 +459,7 @@ public void testBuilder() { .setCrc32c(CRC32) .setCreateTime(CREATE_TIME) .setCustomerEncryption(CUSTOMER_ENCRYPTION) + .setKmsKeyName(KMS_KEY_NAME) .setDeleteTime(DELETE_TIME) .setEtag(ETAG) .setGeneratedId(GENERATED_ID) @@ -511,6 +514,7 @@ public void testBuilder() { assertNull(blob.getCrc32c()); assertNull(blob.getCreateTime()); assertNull(blob.getCustomerEncryption()); + assertNull(blob.getKmsKeyName()); assertNull(blob.getDeleteTime()); assertNull(blob.getEtag()); assertNull(blob.getGeneratedId()); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java index dbab8bc4ad83..cb5dd63d1adb 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java @@ -102,6 +102,8 @@ public class StorageImplTest { private static final String BASE64_KEY = "JVzfVl8NLD9FjedFuStegjRfES5ll5zc59CIXw572OA="; private static final Key KEY = new SecretKeySpec(BaseEncoding.base64().decode(BASE64_KEY), "AES256"); + private static final String KMS_KEY_NAME = + "projects/gcloud-devel/locations/us/keyRings/gcs_kms_key_ring_us/cryptoKeys/key"; // BucketInfo objects private static final BucketInfo BUCKET_INFO1 = @@ -245,6 +247,9 @@ public class StorageImplTest { private static final Map ENCRYPTION_KEY_OPTIONS = ImmutableMap.of(StorageRpc.Option.CUSTOMER_SUPPLIED_KEY, BASE64_KEY); + // Customer managed encryption key options + private static final Map KMS_KEY_NAME_OPTIONS = + ImmutableMap.of(StorageRpc.Option.KMS_KEY_NAME, KMS_KEY_NAME); // IAM policies private static final String POLICY_ETAG1 = "CAE="; private static final String POLICY_ETAG2 = "CAI="; diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index 64efc81f1f05..09360efe2a4f 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -239,12 +239,13 @@ public void testCreateBlobWithEncryptionKey() { @Test public void testCreateBlobWithKmsKeyName() { String blobName = "test-create-with-kms-key-name-blob"; - BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).setKmsKeyName(KMS_KEY_NAME_1).build(); - Blob remoteBlob = storage.create(blob, BLOB_BYTE_CONTENT); + BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); + Blob remoteBlob = storage.create(blob, BLOB_BYTE_CONTENT, Storage.BlobTargetOption.kmsKeyName(KMS_KEY_NAME_1)); assertNotNull(remoteBlob); assertEquals(blob.getBucket(), remoteBlob.getBucket()); assertEquals(blob.getName(), remoteBlob.getName()); - assertTrue(remoteBlob.getKmsKeyName().startsWith(blob.getKmsKeyName())); + assertNotNull(remoteBlob.getKmsKeyName()); + assertTrue(remoteBlob.getKmsKeyName().startsWith(KMS_KEY_NAME_1)); byte[] readBytes = storage.readAllBytes(BUCKET, blobName); assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); } @@ -253,8 +254,9 @@ public void testCreateBlobWithKmsKeyName() { public void testCreateBlobWithKmsKeyNameAndCustomerSuppliedKey() { try { String blobName = "test-create-with-kms-key-name-blob"; - BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).setKmsKeyName(KMS_KEY_NAME_1).build(); - storage.create(blob, BLOB_BYTE_CONTENT, Storage.BlobTargetOption.encryptionKey(KEY)); + BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); + storage.create(blob, BLOB_BYTE_CONTENT, Storage.BlobTargetOption.encryptionKey(KEY), + Storage.BlobTargetOption.kmsKeyName(KMS_KEY_NAME_1)); fail("StorageException was expected"); // can't supply both. } catch (StorageException ex) { // expected @@ -367,6 +369,20 @@ public void testGetBlobSelectedFields() { assertNull(remoteBlob.getContentType()); } + @Test + public void testGetBlobKmsKeyNameField() { + String blobName = "test-get-selected-kms-key-name-field-blob"; + BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName) + .setContentType(CONTENT_TYPE) + .build(); + assertNotNull(storage.create(blob, Storage.BlobTargetOption.kmsKeyName(KMS_KEY_NAME_1))); + Blob remoteBlob = storage.get(blob.getBlobId(), Storage.BlobGetOption.fields( + BlobField.KMS_KEY_NAME)); + assertEquals(blob.getBlobId(), remoteBlob.getBlobId()); + assertTrue(remoteBlob.getKmsKeyName().startsWith(KMS_KEY_NAME_1)); + assertNull(remoteBlob.getContentType()); + } + @Test public void testGetBlobAllSelectedFields() { String blobName = "test-get-all-selected-fields-blob"; @@ -459,14 +475,12 @@ public void testListBlobsKmsKeySelectedFields() throws InterruptedException { "test-list-blobs-selected-field-kms-key-name-blob2"}; BlobInfo blob1 = BlobInfo.newBuilder(BUCKET, blobNames[0]) .setContentType(CONTENT_TYPE) - .setKmsKeyName(KMS_KEY_NAME_1) .build(); BlobInfo blob2 = BlobInfo.newBuilder(BUCKET, blobNames[1]) .setContentType(CONTENT_TYPE) - .setKmsKeyName(KMS_KEY_NAME_1) .build(); - Blob remoteBlob1 = storage.create(blob1); - Blob remoteBlob2 = storage.create(blob2); + Blob remoteBlob1 = storage.create(blob1, Storage.BlobTargetOption.kmsKeyName(KMS_KEY_NAME_1)); + Blob remoteBlob2 = storage.create(blob2, Storage.BlobTargetOption.kmsKeyName(KMS_KEY_NAME_1)); assertNotNull(remoteBlob1); assertNotNull(remoteBlob2); Page page = storage.list(BUCKET, @@ -936,12 +950,11 @@ public void testRotateFromCustomerEncryptionToKmsKey() { BlobInfo target = BlobInfo.newBuilder(BUCKET, targetBlobName) .setContentType(CONTENT_TYPE) .setMetadata(metadata) - .setKmsKeyName(KMS_KEY_NAME_1) .build(); Storage.CopyRequest req = Storage.CopyRequest.newBuilder() .setSource(source) .setSourceOptions(Storage.BlobSourceOption.decryptionKey(BASE64_KEY)) - .setTarget(target) + .setTarget(target, Storage.BlobTargetOption.kmsKeyName(KMS_KEY_NAME_1)) .build(); CopyWriter copyWriter = storage.copy(req); assertEquals(BUCKET, copyWriter.getResult().getBucket()); @@ -967,13 +980,13 @@ public void testRotateFromCustomerEncryptionToKmsKeyWithCustomerEncrytion() { BlobInfo target = BlobInfo.newBuilder(BUCKET, targetBlobName) .setContentType(CONTENT_TYPE) .setMetadata(metadata) - .setKmsKeyName(KMS_KEY_NAME_1) .build(); try { Storage.CopyRequest req = Storage.CopyRequest.newBuilder() .setSource(source) .setSourceOptions(Storage.BlobSourceOption.decryptionKey(BASE64_KEY)) - .setTarget(target, Storage.BlobTargetOption.encryptionKey(KEY)) + .setTarget(target, Storage.BlobTargetOption.encryptionKey(KEY), + Storage.BlobTargetOption.kmsKeyName(KMS_KEY_NAME_1)) .build(); storage.copy(req); fail("StorageException was expected"); @@ -982,7 +995,6 @@ public void testRotateFromCustomerEncryptionToKmsKeyWithCustomerEncrytion() { } } - @Test public void testCopyBlobUpdateMetadata() { String sourceBlobName = "test-copy-blob-update-metadata-source"; @@ -1847,6 +1859,29 @@ public void testListBucketRequesterPaysFails() throws InterruptedException { } } + @Test + public void testListBucketDefaultKmsKeyName() throws InterruptedException { + Bucket remoteBucket = storage.get(BUCKET, Storage.BucketGetOption.fields(BucketField.ENCRYPTION)); + assertNull(remoteBucket.getDefaultKmsKeyName()); + remoteBucket = remoteBucket.toBuilder().setDefaultKmsKeyName(KMS_KEY_NAME_1).build().update(); + assertTrue(remoteBucket.getDefaultKmsKeyName().startsWith(KMS_KEY_NAME_1)); + Iterator bucketIterator = storage.list(Storage.BucketListOption.prefix(BUCKET), + Storage.BucketListOption.fields(BucketField.ENCRYPTION)).iterateAll().iterator(); + while (!bucketIterator.hasNext()) { + Thread.sleep(500); + bucketIterator = storage.list(Storage.BucketListOption.prefix(BUCKET), + Storage.BucketListOption.fields(BucketField.ENCRYPTION)).iterateAll().iterator(); + } + while (bucketIterator.hasNext()) { + Bucket bucket = bucketIterator.next(); + assertTrue(bucket.getName().startsWith(BUCKET)); + assertNotNull(bucket.getDefaultKmsKeyName()); + assertTrue(bucket.getDefaultKmsKeyName().startsWith(KMS_KEY_NAME_1)); + assertNull(bucket.getCreateTime()); + assertNull(bucket.getSelfLink()); + } + } + @Test public void testGetServiceAccount() throws InterruptedException { String projectId = remoteStorageHelper.getOptions().getProjectId(); From 4da3796f6978eda777b9f3d300bf286d8cb92482 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Tue, 8 May 2018 08:15:50 -0700 Subject: [PATCH 09/13] Make setKmsKeyName private --- .../src/main/java/com/google/cloud/storage/Blob.java | 2 +- .../src/main/java/com/google/cloud/storage/BlobInfo.java | 2 +- .../src/main/java/com/google/cloud/storage/Storage.java | 8 +++++++- .../com/google/cloud/storage/spi/v1/HttpStorageRpc.java | 1 - 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java index e7ffbc7515d6..a5d14126e0f6 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java @@ -404,7 +404,7 @@ Builder setCustomerEncryption(CustomerEncryption customerEncryption) { @GcpLaunchStage.Beta @Override - public Builder setKmsKeyName(String kmsKeyName) { + Builder setKmsKeyName(String kmsKeyName) { infoBuilder.setKmsKeyName(kmsKeyName); return this; } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java index c31b61edfe5c..386daa63390a 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java @@ -488,7 +488,7 @@ Builder setCustomerEncryption(CustomerEncryption customerEncryption) { @GcpLaunchStage.Beta @Override - public Builder setKmsKeyName(String kmsKeyName) { + Builder setKmsKeyName(String kmsKeyName) { this.kmsKeyName = kmsKeyName; return this; } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 194b10fea07e..d710c59a0d67 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -22,8 +22,14 @@ import com.google.api.gax.paging.Page; import com.google.auth.ServiceAccountSigner; import com.google.auth.ServiceAccountSigner.SigningException; -import com.google.cloud.*; +import com.google.cloud.FieldSelector; import com.google.cloud.FieldSelector.Helper; +import com.google.cloud.GcpLaunchStage; +import com.google.cloud.Policy; +import com.google.cloud.ReadChannel; +import com.google.cloud.Service; +import com.google.cloud.Tuple; +import com.google.cloud.WriteChannel; import com.google.cloud.storage.Acl.Entity; import com.google.cloud.storage.spi.v1.StorageRpc; import com.google.common.collect.ImmutableList; diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index 68a2e888d29e..1ce683e7aab4 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -81,7 +81,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; - import org.apache.http.HttpStatus; public class HttpStorageRpc implements StorageRpc { From a1b0087ae33739e9eb741680824fba3bb37bd678 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Tue, 8 May 2018 10:49:23 -0700 Subject: [PATCH 10/13] Adding unit tests --- .../com/google/cloud/storage/BlobTest.java | 12 +++ .../com/google/cloud/storage/BucketTest.java | 16 ++++ .../google/cloud/storage/StorageImplTest.java | 87 +++++++++++++++++++ 3 files changed, 115 insertions(+) diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java index a15544f68959..f1b52a526b71 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java @@ -367,6 +367,18 @@ public void testWriterWithEncryptionKey() throws Exception { assertSame(channel, blob.writer(BlobWriteOption.encryptionKey(KEY))); } + @Test + public void testWriterWithKmsKeyName() throws Exception { + initializeExpectedBlob(2); + BlobWriteChannel channel = createMock(BlobWriteChannel.class); + expect(storage.getOptions()).andReturn(mockOptions); + expect(storage.writer(eq(expectedBlob), eq(BlobWriteOption.kmsKeyName(KMS_KEY_NAME)))) + .andReturn(channel); + replay(storage); + initializeBlob(); + assertSame(channel, blob.writer(BlobWriteOption.kmsKeyName(KMS_KEY_NAME))); + } + @Test public void testSignUrl() throws Exception { initializeExpectedBlob(2); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java index 94c8ad715a0a..707a9f3c7565 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java @@ -366,6 +366,22 @@ public void testCreateWithEncryptionKey() throws Exception { assertEquals(expectedBlob, blob); } + @Test + public void testCreateWithKmsKeyName() throws Exception { + initializeExpectedBucket(5); + BlobInfo info = BlobInfo.newBuilder(BlobId.of("b", "n")).setContentType(CONTENT_TYPE).build(); + Blob expectedBlob = new Blob(serviceMockReturnsOptions, new BlobInfo.BuilderImpl(info)); + byte[] content = {0xD, 0xE, 0xA, 0xD}; + expect(storage.getOptions()).andReturn(mockOptions); + expect(storage.create(info, content, Storage.BlobTargetOption.kmsKeyName(DEFAULT_KMS_KEY_NAME))) + .andReturn(expectedBlob); + replay(storage); + initializeBucket(); + Blob blob = + bucket.create("n", content, CONTENT_TYPE, Bucket.BlobTargetOption.kmsKeyName(DEFAULT_KMS_KEY_NAME)); + assertEquals(expectedBlob, blob); + } + @Test public void testCreateNotExists() throws Exception { initializeExpectedBucket(5); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java index cb5dd63d1adb..9fea3887c464 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java @@ -566,6 +566,40 @@ public void testCreateBlobWithEncryptionKey() throws IOException { assertEquals(-1, byteStream.read(streamBytes)); } + @Test + public void testCreateBlobWithKmsKeyName() throws IOException { + Capture capturedStream = Capture.newInstance(); + EasyMock.expect( + storageRpcMock.create( + EasyMock.eq( + BLOB_INFO1 + .toBuilder() + .setMd5(CONTENT_MD5) + .setCrc32c(CONTENT_CRC32C) + .build() + .toPb()), + EasyMock.capture(capturedStream), + EasyMock.eq(KMS_KEY_NAME_OPTIONS))) + .andReturn(BLOB_INFO1.toPb()) + .times(2); + EasyMock.replay(storageRpcMock); + initializeService(); + Blob blob = storage.create(BLOB_INFO1, BLOB_CONTENT, BlobTargetOption.kmsKeyName(KMS_KEY_NAME)); + assertEquals(expectedBlob1, blob); + ByteArrayInputStream byteStream = capturedStream.getValue(); + byte[] streamBytes = new byte[BLOB_CONTENT.length]; + assertEquals(BLOB_CONTENT.length, byteStream.read(streamBytes)); + assertArrayEquals(BLOB_CONTENT, streamBytes); + assertEquals(-1, byteStream.read(streamBytes)); + blob = storage.create(BLOB_INFO1, BLOB_CONTENT, BlobTargetOption.kmsKeyName(KMS_KEY_NAME)); + assertEquals(expectedBlob1, blob); + byteStream = capturedStream.getValue(); + streamBytes = new byte[BLOB_CONTENT.length]; + assertEquals(BLOB_CONTENT.length, byteStream.read(streamBytes)); + assertArrayEquals(BLOB_CONTENT, streamBytes); + assertEquals(-1, byteStream.read(streamBytes)); + } + @Test public void testCreateBlobFromStream() throws IOException { Capture capturedStream = Capture.newInstance(); @@ -1243,6 +1277,43 @@ public void testCopyWithEncryptionKey() { assertTrue(!writer.isDone()); } + @Test + public void testCopyFromEncryptionKeyToKmsKeyName() { + CopyRequest request = + Storage.CopyRequest.newBuilder() + .setSource(BLOB_INFO2.getBlobId()) + .setSourceOptions(BlobSourceOption.decryptionKey(KEY)) + .setTarget(BLOB_INFO1, BlobTargetOption.kmsKeyName(KMS_KEY_NAME)) + .build(); + StorageRpc.RewriteRequest rpcRequest = + new StorageRpc.RewriteRequest( + request.getSource().toPb(), + ENCRYPTION_KEY_OPTIONS, + true, + request.getTarget().toPb(), + KMS_KEY_NAME_OPTIONS, + null); + StorageRpc.RewriteResponse rpcResponse = + new StorageRpc.RewriteResponse(rpcRequest, null, 42L, false, "token", 21L); + EasyMock.expect(storageRpcMock.openRewrite(rpcRequest)).andReturn(rpcResponse).times(2); + EasyMock.replay(storageRpcMock); + initializeService(); + CopyWriter writer = storage.copy(request); + assertEquals(42L, writer.getBlobSize()); + assertEquals(21L, writer.getTotalBytesCopied()); + assertTrue(!writer.isDone()); + request = + Storage.CopyRequest.newBuilder() + .setSource(BLOB_INFO2.getBlobId()) + .setSourceOptions(BlobSourceOption.decryptionKey(BASE64_KEY)) + .setTarget(BLOB_INFO1, BlobTargetOption.kmsKeyName(KMS_KEY_NAME)) + .build(); + writer = storage.copy(request); + assertEquals(42L, writer.getBlobSize()); + assertEquals(21L, writer.getTotalBytesCopied()); + assertTrue(!writer.isDone()); + } + @Test public void testCopyWithOptionsFromBlobId() { CopyRequest request = @@ -1494,6 +1565,22 @@ public void testWriterWithEncryptionKey() { assertTrue(channel.isOpen()); } + @Test + public void testWriterWithKmsKeyName() { + BlobInfo info = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build(); + EasyMock.expect(storageRpcMock.open(info.toPb(), KMS_KEY_NAME_OPTIONS)) + .andReturn("upload-id") + .times(2); + EasyMock.replay(storageRpcMock); + initializeService(); + WriteChannel channel = storage.writer(info, BlobWriteOption.kmsKeyName(KMS_KEY_NAME)); + assertNotNull(channel); + assertTrue(channel.isOpen()); + channel = storage.writer(info, BlobWriteOption.kmsKeyName(KMS_KEY_NAME)); + assertNotNull(channel); + assertTrue(channel.isOpen()); + } + @Test public void testSignUrl() throws NoSuchAlgorithmException, InvalidKeyException, SignatureException, From 619c6e41b95666fb2fdd612b57b1f24420400756 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Tue, 8 May 2018 11:03:38 -0700 Subject: [PATCH 11/13] Adding launch stage annotation to Fields --- .../src/main/java/com/google/cloud/storage/Storage.java | 2 ++ .../main/java/com/google/cloud/storage/spi/v1/StorageRpc.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index d710c59a0d67..b49d48c70b08 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -93,6 +93,7 @@ enum BucketField implements FieldSelector { CORS("cors"), STORAGE_CLASS("storageClass"), ETAG("etag"), + @GcpLaunchStage.Beta ENCRYPTION("encryption"), BILLING("billing"); @@ -135,6 +136,7 @@ enum BlobField implements FieldSelector { SIZE("size"), STORAGE_CLASS("storageClass"), TIME_DELETED("timeDeleted"), + @GcpLaunchStage.Beta KMS_KEY_NAME("kmsKeyName"), UPDATED("updated"); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java index ef0ffcd06cce..780d8b864cfd 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java @@ -25,6 +25,7 @@ import com.google.api.services.storage.model.ServiceAccount; import com.google.api.services.storage.model.StorageObject; import com.google.api.services.storage.model.TestIamPermissionsResponse; +import com.google.cloud.GcpLaunchStage; import com.google.cloud.ServiceRpc; import com.google.cloud.Tuple; import com.google.cloud.storage.StorageException; @@ -56,6 +57,7 @@ enum Option { FIELDS("fields"), CUSTOMER_SUPPLIED_KEY("customerSuppliedKey"), USER_PROJECT("userProject"), + @GcpLaunchStage.Beta KMS_KEY_NAME("kmsKeyName"); private final String value; From 28476093f357cb4b46dc65ddb3d6d7e8bb4cc02b Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Mon, 21 May 2018 12:23:56 -0700 Subject: [PATCH 12/13] Update storage api client library version --- google-cloud-clients/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-clients/pom.xml b/google-cloud-clients/pom.xml index d33a07e6cfc3..0c81c971a006 100644 --- a/google-cloud-clients/pom.xml +++ b/google-cloud-clients/pom.xml @@ -213,7 +213,7 @@ com.google.apis google-api-services-storage - v1-rev114-1.23.0 + v1-rev131-1.23.0 com.google.apis From e7e4911fb1f4b5e324673b7f337bc45a54b1aac2 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Mon, 21 May 2018 15:31:27 -0700 Subject: [PATCH 13/13] Remove String instantiation --- .../src/main/java/com/google/cloud/storage/BucketInfo.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java index e2fe369aa375..6e0b3e2c38e9 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java @@ -599,7 +599,7 @@ public Builder setLabels(Map labels) { @Override public Builder setDefaultKmsKeyName(String defaultKmsKeyName) { this.defaultKmsKeyName = defaultKmsKeyName != null - ? new String(defaultKmsKeyName) : Data.nullOf(String.class); + ? defaultKmsKeyName : Data.nullOf(String.class); return this; }