diff --git a/google-cloud-clients/google-cloud-storage/pom.xml b/google-cloud-clients/google-cloud-storage/pom.xml index 5f68ed6cd242..3a56fd8691da 100644 --- a/google-cloud-clients/google-cloud-storage/pom.xml +++ b/google-cloud-clients/google-cloud-storage/pom.xml @@ -41,6 +41,26 @@ + + io.grpc + grpc-netty-shaded + test + + + io.grpc + grpc-auth + test + + + com.google.api.grpc + grpc-google-cloud-kms-v1 + test + + + com.google.api.grpc + grpc-google-iam-v1 + test + ${project.groupId} google-cloud-core diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index ce873f9ea717..a1330e946738 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -27,11 +27,22 @@ import static org.junit.Assert.fail; import com.google.api.gax.paging.Page; +import com.google.auth.oauth2.GoogleCredentials; +import com.google.cloud.kms.v1.CreateCryptoKeyRequest; +import com.google.cloud.kms.v1.CreateKeyRingRequest; +import com.google.cloud.kms.v1.CryptoKeyName; +import com.google.cloud.kms.v1.CryptoKey; +import com.google.cloud.kms.v1.GetCryptoKeyRequest; +import com.google.cloud.kms.v1.GetKeyRingRequest; +import com.google.cloud.kms.v1.KeyManagementServiceGrpc.KeyManagementServiceBlockingStub; +import com.google.cloud.kms.v1.KeyManagementServiceGrpc; +import com.google.cloud.kms.v1.KeyRingName; import com.google.cloud.Identity; import com.google.cloud.Policy; import com.google.cloud.ReadChannel; import com.google.cloud.RestorableState; import com.google.cloud.WriteChannel; +import com.google.cloud.kms.v1.LocationName; import com.google.cloud.storage.Acl; import com.google.cloud.storage.Acl.Role; import com.google.cloud.storage.Acl.User; @@ -60,6 +71,7 @@ import com.google.common.collect.Lists; import com.google.common.io.BaseEncoding; import com.google.common.io.ByteStreams; +import com.google.iam.v1.Binding; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -68,8 +80,6 @@ import java.net.URLConnection; import java.nio.ByteBuffer; import java.security.Key; -import java.security.NoSuchAlgorithmException; -import java.security.spec.InvalidKeySpecException; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -85,6 +95,17 @@ import java.util.logging.Logger; import java.util.zip.GZIPInputStream; import javax.crypto.spec.SecretKeySpec; + +import com.google.iam.v1.IAMPolicyGrpc; +import com.google.iam.v1.SetIamPolicyRequest; +import io.grpc.auth.MoreCallCredentials; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.Metadata; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.MetadataUtils; + import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -93,7 +114,11 @@ public class ITStorageTest { private static RemoteStorageHelper remoteStorageHelper; private static Storage storage; - + private static String kmsKeyOneResourcePath; + private static String kmsKeyTwoResourcePath; + private static Metadata requestParamsHeader = new Metadata(); + private static Metadata.Key requestParamsKey = + Metadata.Key.of("x-goog-request-params", Metadata.ASCII_STRING_MARSHALLER); private static final Logger log = Logger.getLogger(ITStorageTest.class.getName()); private static final String BUCKET = RemoteStorageHelper.generateBucketName(); private static final String CONTENT_TYPE = "text/plain"; @@ -107,18 +132,23 @@ public class ITStorageTest { private static final byte[] COMPRESSED_CONTENT = BaseEncoding.base64() .decode("H4sIAAAAAAAAAPNIzcnJV3DPz0/PSVVwzskvTVEILskvSkxPVQQA/LySchsAAAA="); private static final Map BUCKET_LABELS = ImmutableMap.of("label1", "value1"); - private static final String SERVICE_ACCOUNT_EMAIL = "gcloud-devel@gs-project-accounts.iam.gserviceaccount.com"; - private static final String KMS_KEY_NAME_1 = "projects/gcloud-devel/locations/us/keyRings/gcs_kms_key_ring_us/cryptoKeys/key"; - private static final String KMS_KEY_NAME_2 = "projects/gcloud-devel/locations/us/keyRings/gcs_kms_key_ring_us/cryptoKeys/key2"; + private static final String SERVICE_ACCOUNT_EMAIL_SUFFIX = "@gs-project-accounts.iam.gserviceaccount.com"; + private static final String KMS_KEY_RING_NAME = "gcs_test_kms_key_ring"; + private static final String KMS_KEY_RING_LOCATION = "us"; + private static final String KMS_KEY_ONE_NAME = "gcs_kms_key_one"; + private static final String KMS_KEY_TWO_NAME = "gcs_kms_key_two"; @BeforeClass - public static void beforeClass() throws NoSuchAlgorithmException, InvalidKeySpecException { + public static void beforeClass() throws IOException { remoteStorageHelper = RemoteStorageHelper.create(); storage = remoteStorageHelper.getOptions().getService(); storage.create( BucketInfo.newBuilder(BUCKET) .setDeleteRules(Collections.singleton(new BucketInfo.AgeDeleteRule(1))) .build()); + + // Prepare KMS KeyRing for CMEK tests + prepareKmsKeys(); } @AfterClass @@ -137,6 +167,98 @@ public static void afterClass() throws ExecutionException, InterruptedException } } + private static void prepareKmsKeys() throws IOException { + String projectId = remoteStorageHelper.getOptions().getProjectId(); + GoogleCredentials credentials = GoogleCredentials.getApplicationDefault(); + ManagedChannel kmsChannel = ManagedChannelBuilder.forTarget("cloudkms.googleapis.com:443").build(); + KeyManagementServiceBlockingStub kmsStub = KeyManagementServiceGrpc.newBlockingStub(kmsChannel) + .withCallCredentials(MoreCallCredentials.from(credentials)); + IAMPolicyGrpc.IAMPolicyBlockingStub iamStub = IAMPolicyGrpc.newBlockingStub(kmsChannel) + .withCallCredentials(MoreCallCredentials.from(credentials)); + ensureKmsKeyRingExistsForTests(kmsStub, projectId, KMS_KEY_RING_LOCATION, KMS_KEY_RING_NAME); + ensureKmsKeyRingIamPermissionsForTests(iamStub, projectId, KMS_KEY_RING_LOCATION, KMS_KEY_RING_NAME); + kmsKeyOneResourcePath = ensureKmsKeyExistsForTests(kmsStub, projectId, KMS_KEY_RING_LOCATION, KMS_KEY_RING_NAME, + KMS_KEY_ONE_NAME); + kmsKeyTwoResourcePath = ensureKmsKeyExistsForTests(kmsStub, projectId, KMS_KEY_RING_LOCATION, KMS_KEY_RING_NAME, + KMS_KEY_TWO_NAME); + } + + private static String ensureKmsKeyRingExistsForTests(KeyManagementServiceBlockingStub kmsStub, String projectId, + String location, + String keyRingName) throws StatusRuntimeException { + String kmsKeyRingResourcePath = KeyRingName.of(projectId, location, keyRingName).toString(); + try { + // Attempt to Get KeyRing + GetKeyRingRequest getKeyRingRequest = GetKeyRingRequest.newBuilder().setName(kmsKeyRingResourcePath) + .build(); + requestParamsHeader.put(requestParamsKey, "name="+kmsKeyRingResourcePath); + KeyManagementServiceBlockingStub stubForGetKeyRing = MetadataUtils + .attachHeaders(kmsStub, requestParamsHeader); + stubForGetKeyRing.getKeyRing(getKeyRingRequest); + } catch (StatusRuntimeException ex) { + if (ex.getStatus().getCode() == Status.Code.NOT_FOUND) { + // Create KmsKeyRing + String keyRingParent = LocationName.of(projectId, location).toString(); + CreateKeyRingRequest createKeyRingRequest = CreateKeyRingRequest.newBuilder() + .setParent(keyRingParent) + .setKeyRingId(keyRingName).build(); + requestParamsHeader.put(requestParamsKey, "parent=" + keyRingParent); + KeyManagementServiceBlockingStub stubForCreateKeyRing = MetadataUtils + .attachHeaders(kmsStub, requestParamsHeader); + stubForCreateKeyRing.createKeyRing(createKeyRingRequest); + } else { + throw ex; + } + } + + return kmsKeyRingResourcePath; + } + + private static void ensureKmsKeyRingIamPermissionsForTests(IAMPolicyGrpc.IAMPolicyBlockingStub iamStub, + String projectId, String location, + String keyRingName) throws StatusRuntimeException { + ServiceAccount serviceAccount = storage.getServiceAccount(projectId); + String kmsKeyRingResourcePath = KeyRingName.of(projectId, location, keyRingName).toString(); + Binding binding = Binding.newBuilder().setRole("roles/cloudkms.cryptoKeyEncrypterDecrypter") + .addMembers("serviceAccount:"+serviceAccount.getEmail()).build(); + com.google.iam.v1.Policy policy = com.google.iam.v1.Policy.newBuilder().addBindings(binding).build(); + SetIamPolicyRequest setIamPolicyRequest = SetIamPolicyRequest.newBuilder().setResource(kmsKeyRingResourcePath) + .setPolicy(policy).build(); + requestParamsHeader.put(requestParamsKey, "parent=" + kmsKeyRingResourcePath); + iamStub = MetadataUtils.attachHeaders(iamStub, requestParamsHeader); + iamStub.setIamPolicy(setIamPolicyRequest); + } + + private static String ensureKmsKeyExistsForTests(KeyManagementServiceBlockingStub kmsStub, String projectId, + String location, String keyRingName, + String keyName) throws StatusRuntimeException { + String kmsKeyResourcePath = CryptoKeyName.of(projectId, location, keyRingName, keyName).toString(); + try { + // Attempt to Get CryptoKey + requestParamsHeader.put(requestParamsKey, "name=" + kmsKeyResourcePath); + GetCryptoKeyRequest getCryptoKeyRequest = GetCryptoKeyRequest.newBuilder() + .setName(kmsKeyResourcePath).build(); + KeyManagementServiceGrpc.KeyManagementServiceBlockingStub stubForGetCryptoKey = MetadataUtils + .attachHeaders(kmsStub, requestParamsHeader); + stubForGetCryptoKey.getCryptoKey(getCryptoKeyRequest); + } catch(StatusRuntimeException ex) { + if (ex.getStatus().getCode() == Status.Code.NOT_FOUND) { + String kmsKeyRingResourcePath = KeyRingName.of(projectId, location, keyRingName).toString(); + CryptoKey cryptoKey = CryptoKey.newBuilder().setPurpose(CryptoKey.CryptoKeyPurpose.ENCRYPT_DECRYPT).build(); + CreateCryptoKeyRequest createCryptoKeyRequest = CreateCryptoKeyRequest.newBuilder() + .setCryptoKeyId(keyName).setParent(kmsKeyRingResourcePath).setCryptoKey(cryptoKey).build(); + + requestParamsHeader.put(requestParamsKey, "parent=" + kmsKeyRingResourcePath); + KeyManagementServiceGrpc.KeyManagementServiceBlockingStub stubForCreateCryptoKey = MetadataUtils + .attachHeaders(kmsStub, requestParamsHeader); + stubForCreateCryptoKey.createCryptoKey(createCryptoKeyRequest); + } else { + throw ex; + } + } + return kmsKeyResourcePath; + } + @Test(timeout = 5000) public void testListBuckets() throws InterruptedException { Iterator bucketIterator = storage.list(Storage.BucketListOption.prefix(BUCKET), @@ -183,10 +305,10 @@ public void testGetBucketEmptyFields() { public void testClearBucketDefaultKmsKeyName() throws ExecutionException, InterruptedException { String bucketName = RemoteStorageHelper.generateBucketName(); Bucket remoteBucket = storage.create(BucketInfo.newBuilder(bucketName) - .setDefaultKmsKeyName(KMS_KEY_NAME_1).setLocation("US").build()); + .setDefaultKmsKeyName(kmsKeyOneResourcePath).setLocation(KMS_KEY_RING_LOCATION).build()); try { - assertEquals(KMS_KEY_NAME_1, remoteBucket.getDefaultKmsKeyName()); + assertEquals(kmsKeyOneResourcePath, remoteBucket.getDefaultKmsKeyName()); Bucket updatedBucket = remoteBucket.toBuilder().setDefaultKmsKeyName(null).build().update(); assertNull(updatedBucket.getDefaultKmsKeyName()); } finally { @@ -198,12 +320,12 @@ public void testClearBucketDefaultKmsKeyName() throws ExecutionException, Interr public void testUpdateBucketDefaultKmsKeyName() throws ExecutionException, InterruptedException { String bucketName = RemoteStorageHelper.generateBucketName(); Bucket remoteBucket = storage.create(BucketInfo.newBuilder(bucketName) - .setDefaultKmsKeyName(KMS_KEY_NAME_1).setLocation("US").build()); + .setDefaultKmsKeyName(kmsKeyOneResourcePath).setLocation(KMS_KEY_RING_LOCATION).build()); try { - assertEquals(KMS_KEY_NAME_1, remoteBucket.getDefaultKmsKeyName()); - Bucket updatedBucket = remoteBucket.toBuilder().setDefaultKmsKeyName(KMS_KEY_NAME_2).build().update(); - assertEquals(KMS_KEY_NAME_2, updatedBucket.getDefaultKmsKeyName()); + assertEquals(kmsKeyOneResourcePath, remoteBucket.getDefaultKmsKeyName()); + Bucket updatedBucket = remoteBucket.toBuilder().setDefaultKmsKeyName(kmsKeyTwoResourcePath).build().update(); + assertEquals(kmsKeyTwoResourcePath, updatedBucket.getDefaultKmsKeyName()); } finally { RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); } @@ -240,12 +362,12 @@ public void testCreateBlobWithEncryptionKey() { public void testCreateBlobWithKmsKeyName() { String blobName = "test-create-with-kms-key-name-blob"; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); - Blob remoteBlob = storage.create(blob, BLOB_BYTE_CONTENT, Storage.BlobTargetOption.kmsKeyName(KMS_KEY_NAME_1)); + Blob remoteBlob = storage.create(blob, BLOB_BYTE_CONTENT, Storage.BlobTargetOption.kmsKeyName(kmsKeyOneResourcePath)); assertNotNull(remoteBlob); assertEquals(blob.getBucket(), remoteBlob.getBucket()); assertEquals(blob.getName(), remoteBlob.getName()); assertNotNull(remoteBlob.getKmsKeyName()); - assertTrue(remoteBlob.getKmsKeyName().startsWith(KMS_KEY_NAME_1)); + assertTrue(remoteBlob.getKmsKeyName().startsWith(kmsKeyOneResourcePath)); byte[] readBytes = storage.readAllBytes(BUCKET, blobName); assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); } @@ -256,7 +378,7 @@ public void testCreateBlobWithKmsKeyNameAndCustomerSuppliedKey() { String blobName = "test-create-with-kms-key-name-blob"; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); storage.create(blob, BLOB_BYTE_CONTENT, Storage.BlobTargetOption.encryptionKey(KEY), - Storage.BlobTargetOption.kmsKeyName(KMS_KEY_NAME_1)); + Storage.BlobTargetOption.kmsKeyName(kmsKeyOneResourcePath)); fail("StorageException was expected"); // can't supply both. } catch (StorageException ex) { // expected @@ -267,8 +389,8 @@ public void testCreateBlobWithKmsKeyNameAndCustomerSuppliedKey() { public void testCreateBlobWithDefaultKmsKeyName() throws ExecutionException, InterruptedException { String bucketName = RemoteStorageHelper.generateBucketName(); Bucket bucket = storage.create(BucketInfo.newBuilder(bucketName) - .setDefaultKmsKeyName(KMS_KEY_NAME_1).setLocation("US").build()); - assertEquals(bucket.getDefaultKmsKeyName(), KMS_KEY_NAME_1); + .setDefaultKmsKeyName(kmsKeyOneResourcePath).setLocation(KMS_KEY_RING_LOCATION).build()); + assertEquals(bucket.getDefaultKmsKeyName(), kmsKeyOneResourcePath); try { String blobName = "test-create-with-default-kms-key-name-blob"; @@ -278,7 +400,7 @@ public void testCreateBlobWithDefaultKmsKeyName() throws ExecutionException, Int assertEquals(blob.getBucket(), remoteBlob.getBucket()); assertEquals(blob.getName(), remoteBlob.getName()); assertNotNull(remoteBlob.getKmsKeyName()); - assertTrue(remoteBlob.getKmsKeyName().startsWith(KMS_KEY_NAME_1)); + assertTrue(remoteBlob.getKmsKeyName().startsWith(kmsKeyOneResourcePath)); byte[] readBytes = storage.readAllBytes(bucketName, blobName); assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); } finally { @@ -375,11 +497,11 @@ public void testGetBlobKmsKeyNameField() { BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName) .setContentType(CONTENT_TYPE) .build(); - assertNotNull(storage.create(blob, Storage.BlobTargetOption.kmsKeyName(KMS_KEY_NAME_1))); + assertNotNull(storage.create(blob, Storage.BlobTargetOption.kmsKeyName(kmsKeyOneResourcePath))); Blob remoteBlob = storage.get(blob.getBlobId(), Storage.BlobGetOption.fields( BlobField.KMS_KEY_NAME)); assertEquals(blob.getBlobId(), remoteBlob.getBlobId()); - assertTrue(remoteBlob.getKmsKeyName().startsWith(KMS_KEY_NAME_1)); + assertTrue(remoteBlob.getKmsKeyName().startsWith(kmsKeyOneResourcePath)); assertNull(remoteBlob.getContentType()); } @@ -479,8 +601,8 @@ public void testListBlobsKmsKeySelectedFields() throws InterruptedException { BlobInfo blob2 = BlobInfo.newBuilder(BUCKET, blobNames[1]) .setContentType(CONTENT_TYPE) .build(); - Blob remoteBlob1 = storage.create(blob1, Storage.BlobTargetOption.kmsKeyName(KMS_KEY_NAME_1)); - Blob remoteBlob2 = storage.create(blob2, Storage.BlobTargetOption.kmsKeyName(KMS_KEY_NAME_1)); + Blob remoteBlob1 = storage.create(blob1, Storage.BlobTargetOption.kmsKeyName(kmsKeyOneResourcePath)); + Blob remoteBlob2 = storage.create(blob2, Storage.BlobTargetOption.kmsKeyName(kmsKeyOneResourcePath)); assertNotNull(remoteBlob1); assertNotNull(remoteBlob2); Page page = storage.list(BUCKET, @@ -500,7 +622,7 @@ public void testListBlobsKmsKeySelectedFields() throws InterruptedException { Blob remoteBlob = iterator.next(); assertEquals(BUCKET, remoteBlob.getBucket()); assertTrue(blobSet.contains(remoteBlob.getName())); - assertTrue(remoteBlob.getKmsKeyName().startsWith(KMS_KEY_NAME_1)); + assertTrue(remoteBlob.getKmsKeyName().startsWith(kmsKeyOneResourcePath)); assertNull(remoteBlob.getContentType()); } } @@ -982,14 +1104,14 @@ public void testRotateFromCustomerEncryptionToKmsKey() { Storage.CopyRequest req = Storage.CopyRequest.newBuilder() .setSource(source) .setSourceOptions(Storage.BlobSourceOption.decryptionKey(BASE64_KEY)) - .setTarget(target, Storage.BlobTargetOption.kmsKeyName(KMS_KEY_NAME_1)) + .setTarget(target, Storage.BlobTargetOption.kmsKeyName(kmsKeyOneResourcePath)) .build(); CopyWriter copyWriter = storage.copy(req); assertEquals(BUCKET, copyWriter.getResult().getBucket()); assertEquals(targetBlobName, copyWriter.getResult().getName()); assertEquals(CONTENT_TYPE, copyWriter.getResult().getContentType()); assertNotNull(copyWriter.getResult().getKmsKeyName()); - assertTrue(copyWriter.getResult().getKmsKeyName().startsWith(KMS_KEY_NAME_1)); + assertTrue(copyWriter.getResult().getKmsKeyName().startsWith(kmsKeyOneResourcePath)); assertArrayEquals(BLOB_BYTE_CONTENT, copyWriter.getResult().getContent()); assertEquals(metadata, copyWriter.getResult().getMetadata()); assertTrue(copyWriter.isDone()); @@ -1014,7 +1136,7 @@ public void testRotateFromCustomerEncryptionToKmsKeyWithCustomerEncrytion() { .setSource(source) .setSourceOptions(Storage.BlobSourceOption.decryptionKey(BASE64_KEY)) .setTarget(target, Storage.BlobTargetOption.encryptionKey(KEY), - Storage.BlobTargetOption.kmsKeyName(KMS_KEY_NAME_1)) + Storage.BlobTargetOption.kmsKeyName(kmsKeyOneResourcePath)) .build(); storage.copy(req); fail("StorageException was expected"); @@ -1891,9 +2013,9 @@ public void testListBucketRequesterPaysFails() throws InterruptedException { public void testListBucketDefaultKmsKeyName() throws ExecutionException, InterruptedException { String bucketName = RemoteStorageHelper.generateBucketName(); Bucket remoteBucket = storage.create(BucketInfo.newBuilder(bucketName) - .setDefaultKmsKeyName(KMS_KEY_NAME_1).setLocation("US").build()); + .setDefaultKmsKeyName(kmsKeyOneResourcePath).setLocation(KMS_KEY_RING_LOCATION).build()); assertNotNull(remoteBucket); - assertTrue(remoteBucket.getDefaultKmsKeyName().startsWith(KMS_KEY_NAME_1)); + assertTrue(remoteBucket.getDefaultKmsKeyName().startsWith(kmsKeyOneResourcePath)); try { Iterator bucketIterator = storage.list(Storage.BucketListOption.prefix(bucketName), Storage.BucketListOption.fields(BucketField.ENCRYPTION)).iterateAll().iterator(); @@ -1906,7 +2028,7 @@ public void testListBucketDefaultKmsKeyName() throws ExecutionException, Interru Bucket bucket = bucketIterator.next(); assertTrue(bucket.getName().startsWith(bucketName)); assertNotNull(bucket.getDefaultKmsKeyName()); - assertTrue(bucket.getDefaultKmsKeyName().startsWith(KMS_KEY_NAME_1)); + assertTrue(bucket.getDefaultKmsKeyName().startsWith(kmsKeyOneResourcePath)); assertNull(bucket.getCreateTime()); assertNull(bucket.getSelfLink()); } @@ -1920,6 +2042,6 @@ public void testGetServiceAccount() throws InterruptedException { String projectId = remoteStorageHelper.getOptions().getProjectId(); ServiceAccount serviceAccount = storage.getServiceAccount(projectId); assertNotNull(serviceAccount); - assertEquals(SERVICE_ACCOUNT_EMAIL, serviceAccount.getEmail()); + assertTrue(serviceAccount.getEmail().endsWith(SERVICE_ACCOUNT_EMAIL_SUFFIX)); } }