Skip to content

[Storage] KMS integration #3196

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
frankyn merged 14 commits into from
May 21, 2018
Merged

[Storage] KMS integration #3196

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
031b102
Adding base GCS CMEK implementation from rossjudson@
Apr 26, 2018
56b34df
Adding Integration tests and partial unit test coverage
May 1, 2018
47701ed
Fix kms key reosurce name
May 1, 2018
86830e5
Intermediate stage with WriteOptions
May 7, 2018
8d042d2
Clean up and add tests
May 7, 2018
116220c
Address comments
May 7, 2018
afd6eb3
Update storage apiary client library version
May 7, 2018
fa6b526
Use BlobTargetOption and BlobWriteOption
May 8, 2018
4da3796
Make setKmsKeyName private
May 8, 2018
a1b0087
Adding unit tests
May 8, 2018
619c6e4
Adding launch stage annotation to Fields
May 8, 2018
dbc9f2d
Merge branch 'master' into kms-integration
May 21, 2018
2847609
Update storage api client library version
May 21, 2018
e7e4911
Remove String instantiation
May 21, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,6 @@ public static BlobSourceOption decryptionKey(String key) {
* Returns an option for blob's billing user project. This option is used only if the blob's
* bucket has requester_pays flag enabled.
*/
@GcpLaunchStage.Alpha
public static BlobSourceOption userProject(String userProject) {
return new BlobSourceOption(StorageRpc.Option.USER_PROJECT, userProject);
}
Expand Down Expand Up @@ -403,6 +402,12 @@ Builder setCustomerEncryption(CustomerEncryption customerEncryption) {
return this;
}

@Override
public Builder setKmsKeyName(String kmsKeyName) {

This comment was marked as spam.

Sign in to view

This comment was marked as spam.

Sign in to view

infoBuilder.setKmsKeyName(kmsKeyName);
return this;
}

@Override
public Blob build() {
return new Blob(storage, infoBuilder);
Expand Down
29 changes: 29 additions & 0 deletions google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ public StorageObject apply(BlobInfo blobInfo) {
private final Integer componentCount;
private final boolean isDirectory;
private final CustomerEncryption customerEncryption;
private final String kmsKeyName;

/**
* This class is meant for internal use only. Users are discouraged from using this class.
Expand Down Expand Up @@ -266,6 +267,12 @@ public abstract static class Builder {

abstract Builder setCustomerEncryption(CustomerEncryption customerEncryption);

/**
*
* Sets the blob's kmsKeyName.
*/
public abstract Builder setKmsKeyName(String kmsKeyName);

/**
* Creates a {@code BlobInfo} object.
*/
Expand Down Expand Up @@ -298,6 +305,7 @@ static final class BuilderImpl extends Builder {
private Boolean isDirectory;
private CustomerEncryption customerEncryption;
private StorageClass storageClass;
private String kmsKeyName;

BuilderImpl(BlobId blobId) {
this.blobId = blobId;
Expand Down Expand Up @@ -328,6 +336,7 @@ static final class BuilderImpl extends Builder {
createTime = blobInfo.createTime;
isDirectory = blobInfo.isDirectory;
storageClass = blobInfo.storageClass;
kmsKeyName = blobInfo.kmsKeyName;
}

@Override
Expand Down Expand Up @@ -475,6 +484,12 @@ Builder setCustomerEncryption(CustomerEncryption customerEncryption) {
return this;
}

@Override
public Builder setKmsKeyName(String kmsKeyName) {
this.kmsKeyName = kmsKeyName;
return this;
}

@Override
public BlobInfo build() {
checkNotNull(blobId);
Expand Down Expand Up @@ -507,6 +522,7 @@ public BlobInfo build() {
createTime = builder.createTime;
isDirectory = firstNonNull(builder.isDirectory, Boolean.FALSE);
storageClass = builder.storageClass;
kmsKeyName = builder.kmsKeyName;
}

/**
Expand Down Expand Up @@ -737,6 +753,13 @@ public StorageClass getStorageClass() {
return storageClass;
}

/**
* Returns the Cloud KMS key used to encrypt the blob, if any.
*/
public String getKmsKeyName() {
return kmsKeyName;
}

/**
* Returns a builder for the current blob.
*/
Expand Down Expand Up @@ -809,6 +832,9 @@ public ObjectAccessControl apply(Acl acl) {
if (customerEncryption != null) {
storageObject.setCustomerEncryption(customerEncryption.toPb());
}
if (kmsKeyName != null) {
storageObject.setKmsKeyName(kmsKeyName);
}
storageObject.setMetadata(pbMetadata);
storageObject.setCacheControl(cacheControl);
storageObject.setContentEncoding(contentEncoding);
Expand Down Expand Up @@ -939,6 +965,9 @@ public Acl apply(ObjectAccessControl objectAccessControl) {
if (storageObject.getStorageClass() != null) {
builder.setStorageClass(StorageClass.valueOf(storageObject.getStorageClass()));
}
if (storageObject.getKmsKeyName() != null) {
builder.setKmsKeyName(storageObject.getKmsKeyName());
}
return builder.build();
}
}
9 changes: 6 additions & 3 deletions google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,6 @@ public static BucketSourceOption metagenerationNotMatch() {
* Returns an option for blob's billing user project. This option is only used by the buckets with
* 'requester_pays' flag.
*/
@GcpLaunchStage.Alpha
public static BucketSourceOption userProject(String userProject) {

This comment was marked as spam.

Sign in to view

This comment was marked as spam.

Sign in to view

This comment was marked as spam.

Sign in to view

return new BucketSourceOption(StorageRpc.Option.USER_PROJECT, userProject);
}
Expand Down Expand Up @@ -271,7 +270,6 @@ public static BlobTargetOption encryptionKey(String key) {
* Returns an option for blob's billing user project. This option is only used by the buckets with
* 'requester_pays' flag.
*/
@GcpLaunchStage.Alpha
public static BlobTargetOption userProject(String userProject) {
return new BlobTargetOption(StorageRpc.Option.USER_PROJECT, userProject);
}
Expand Down Expand Up @@ -468,7 +466,6 @@ public static BlobWriteOption encryptionKey(String key) {
* Returns an option for blob's billing user project. This option is only used by the buckets with
* 'requester_pays' flag.
*/
@GcpLaunchStage.Alpha
public static BlobWriteOption userProject(String userProject) {
return new BlobWriteOption(Storage.BlobWriteOption.Option.USER_PROJECT, userProject);
}
Expand Down Expand Up @@ -615,6 +612,12 @@ public Builder setLabels(Map<String, String> labels) {
return this;
}

@Override
public Builder setDefaultKmsKeyName(String defaultKmsKeyName) {
infoBuilder.setDefaultKmsKeyName(defaultKmsKeyName);
return this;
}

@Override
public Bucket build() {
return new Bucket(storage, infoBuilder);
Expand Down
29 changes: 29 additions & 0 deletions google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.google.api.client.util.DateTime;
import com.google.api.services.storage.model.*;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Bucket.Encryption;
import com.google.api.services.storage.model.Bucket.Lifecycle;
import com.google.api.services.storage.model.Bucket.Lifecycle.Rule;
import com.google.api.services.storage.model.Bucket.Owner;
Expand Down Expand Up @@ -85,6 +86,7 @@ public com.google.api.services.storage.model.Bucket apply(BucketInfo bucketInfo)
private final String location;
private final StorageClass storageClass;
private final Map<String, String> labels;
private final String defaultKmsKeyName;

/**
* Base class for bucket's delete rules. Allows to configure automatic deletion of blobs and blobs
Expand Down Expand Up @@ -423,6 +425,11 @@ public abstract static class Builder {
*/
public abstract Builder setLabels(Map<String, String> labels);

/**
* Sets the default Cloud KMS key name for this bucket.
*/
public abstract Builder setDefaultKmsKeyName(String defaultKmsKeyName);

/**
* Creates a {@code BucketInfo} object.
*/
Expand All @@ -449,6 +456,7 @@ static final class BuilderImpl extends Builder {
private List<Acl> acl;
private List<Acl> defaultAcl;
private Map<String, String> labels;
private String defaultKmsKeyName;

BuilderImpl(String name) {
this.name = name;
Expand All @@ -473,6 +481,7 @@ static final class BuilderImpl extends Builder {
deleteRules = bucketInfo.deleteRules;
labels = bucketInfo.labels;
requesterPays = bucketInfo.requesterPays;
defaultKmsKeyName = bucketInfo.defaultKmsKeyName;
}

@Override
Expand Down Expand Up @@ -584,6 +593,12 @@ public Builder setLabels(Map<String, String> labels) {
return this;
}

@Override
public Builder setDefaultKmsKeyName(String defaultKmsKeyName) {
this.defaultKmsKeyName = defaultKmsKeyName;
return this;
}

@Override
public BucketInfo build() {
checkNotNull(name);
Expand All @@ -610,6 +625,7 @@ public BucketInfo build() {
deleteRules = builder.deleteRules;
labels = builder.labels;
requesterPays = builder.requesterPays;
defaultKmsKeyName = builder.defaultKmsKeyName;
}

/**
Expand Down Expand Up @@ -762,6 +778,13 @@ public Map<String, String> getLabels() {
return labels;
}

/**
* Returns the default Cloud KMS key to be applied to newly inserted objects in this bucket.

This comment was marked as spam.

Sign in to view

This comment was marked as spam.

Sign in to view

This comment was marked as spam.

Sign in to view

*/
public String getDefaultKmsKeyName() {
return defaultKmsKeyName;
}

/**
* Returns a builder for the current bucket.
*/
Expand Down Expand Up @@ -857,6 +880,8 @@ public Rule apply(DeleteRule deleteRule) {
if (labels != null) {
bucketPb.setLabels(labels);
}
// default kms key name can be null.
bucketPb.setEncryption(new Encryption().setDefaultKmsKeyName(defaultKmsKeyName));

return bucketPb;
}
Expand Down Expand Up @@ -945,6 +970,10 @@ public DeleteRule apply(Rule rule) {
if (billing != null) {
builder.setRequesterPays(billing.getRequesterPays());
}
Encryption encryption = bucketPb.getEncryption();
if (encryption != null && encryption.getDefaultKmsKeyName() != null && !encryption.getDefaultKmsKeyName().isEmpty()) {
builder.setDefaultKmsKeyName(encryption.getDefaultKmsKeyName());

This comment was marked as spam.

Sign in to view

}
return builder.build();
}
}
18 changes: 2 additions & 16 deletions google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ enum BucketField implements FieldSelector {
CORS("cors"),
STORAGE_CLASS("storageClass"),
ETAG("etag"),
@GcpLaunchStage.Alpha
DEFAULT_KMS_KEY_NAME("defaultKmsKeyName"),
BILLING("billing");

static final List<? extends FieldSelector> REQUIRED_FIELDS = ImmutableList.of(NAME);
Expand Down Expand Up @@ -136,6 +136,7 @@ enum BlobField implements FieldSelector {
SIZE("size"),
STORAGE_CLASS("storageClass"),
TIME_DELETED("timeDeleted"),
KMS_KEY_NAME("kmsKeyName"),
UPDATED("updated");

static final List<? extends FieldSelector> REQUIRED_FIELDS = ImmutableList.of(BUCKET, NAME);
Expand Down Expand Up @@ -203,7 +204,6 @@ public static BucketTargetOption metagenerationNotMatch() {
* Returns an option to define the billing user project. This option is required by buckets with
* `requester_pays` flag enabled to assign operation costs.
*/
@GcpLaunchStage.Alpha
public static BucketTargetOption userProject(String userProject) {
return new BucketTargetOption(StorageRpc.Option.USER_PROJECT, userProject);
}
Expand Down Expand Up @@ -240,7 +240,6 @@ public static BucketSourceOption metagenerationNotMatch(long metageneration) {
* Returns an option for bucket's billing user project. This option is only used by the buckets with
* 'requester_pays' flag.
*/
@GcpLaunchStage.Alpha
public static BucketSourceOption userProject(String userProject) {
return new BucketSourceOption(StorageRpc.Option.USER_PROJECT, userProject);
}
Expand Down Expand Up @@ -281,7 +280,6 @@ public static BucketGetOption metagenerationNotMatch(long metageneration) {
* Returns an option for bucket's billing user project. This option is only used by the buckets with
* 'requester_pays' flag.
*/
@GcpLaunchStage.Alpha
public static BucketGetOption userProject(String userProject) {
return new BucketGetOption(StorageRpc.Option.USER_PROJECT, userProject);
}
Expand Down Expand Up @@ -372,7 +370,6 @@ public static BlobTargetOption encryptionKey(Key key) {
* Returns an option for blob's billing user project. This option is only used by the buckets with
* 'requester_pays' flag.
*/
@GcpLaunchStage.Alpha
public static BlobTargetOption userProject(String userProject) {
return new BlobTargetOption(StorageRpc.Option.USER_PROJECT, userProject);
}
Expand Down Expand Up @@ -542,7 +539,6 @@ public static BlobWriteOption encryptionKey(String key) {
* Returns an option for blob's billing user project. This option is only used by the buckets with
* 'requester_pays' flag.
*/
@GcpLaunchStage.Alpha
public static BlobWriteOption userProject(String userProject) {
return new BlobWriteOption(Option.USER_PROJECT, userProject);
}
Expand Down Expand Up @@ -636,7 +632,6 @@ public static BlobSourceOption decryptionKey(String key) {
* Returns an option for blob's billing user project. This option is only used by the buckets with
* 'requester_pays' flag.
*/
@GcpLaunchStage.Alpha
public static BlobSourceOption userProject(String userProject) {
return new BlobSourceOption(StorageRpc.Option.USER_PROJECT, userProject);
}
Expand Down Expand Up @@ -726,7 +721,6 @@ public static BlobGetOption fields(BlobField... fields) {
* Returns an option for blob's billing user project. This option is only used by the buckets with
* 'requester_pays' flag.
*/
@GcpLaunchStage.Alpha
public static BlobGetOption userProject(String userProject) {
return new BlobGetOption(StorageRpc.Option.USER_PROJECT, userProject);
}
Expand Down Expand Up @@ -769,7 +763,6 @@ public static BucketListOption prefix(String prefix) {
* Returns an option for bucket's billing user project. This option is only used by the buckets with
* 'requester_pays' flag.
*/
@GcpLaunchStage.Alpha
public static BucketListOption userProject(String userProject) {
return new BucketListOption(StorageRpc.Option.USER_PROJECT, userProject);
}
Expand Down Expand Up @@ -840,7 +833,6 @@ public static BlobListOption currentDirectory() {
*
* @param userProject projectId of the billing user project.
*/
@GcpLaunchStage.Alpha
public static BlobListOption userProject(String userProject) {
return new BlobListOption(StorageRpc.Option.USER_PROJECT, userProject);
}
Expand Down Expand Up @@ -2587,8 +2579,6 @@ public static Builder newBuilder() {
* @param options extra parameters to apply to this operation
* @throws StorageException upon failure
*/
@BetaApi
@GcpLaunchStage.Alpha
Policy getIamPolicy(String bucket, BucketSourceOption... options);

/**
Expand All @@ -2612,8 +2602,6 @@ public static Builder newBuilder() {
* @param options extra parameters to apply to this operation
* @throws StorageException upon failure
*/
@BetaApi
@GcpLaunchStage.Alpha
Policy setIamPolicy(String bucket, Policy policy, BucketSourceOption... options);

/**
Expand All @@ -2637,8 +2625,6 @@ public static Builder newBuilder() {
* @param options extra parameters to apply to this operation
* @throws StorageException upon failure
*/
@BetaApi
@GcpLaunchStage.Alpha
List<Boolean> testIamPermissions(String bucket, List<String> permissions, BucketSourceOption... options);

/**
Expand Down
Loading