Skip to content

Commit 2313d91

Browse files
ahmarsuhailahmarsuhail
committed
adds CSE materials class
1 parent d787b91 commit 2313d91

5 files changed

Lines changed: 121 additions & 52 deletions

File tree

‎hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java‎

Lines changed: 19 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -40,16 +40,15 @@
4040
import software.amazon.awssdk.services.s3.S3Configuration;
4141
import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration;
4242
import software.amazon.awssdk.transfer.s3.S3TransferManager;
43-
import software.amazon.awssdk.services.kms.KmsClient;
4443
import software.amazon.encryption.s3.S3AsyncEncryptionClient;
4544
import software.amazon.encryption.s3.S3EncryptionClient;
46-
import software.amazon.encryption.s3.materials.KmsKeyring;
4745

4846
import org.apache.commons.lang3.StringUtils;
4947
import org.apache.hadoop.classification.InterfaceAudience;
5048
import org.apache.hadoop.classification.InterfaceStability;
5149
import org.apache.hadoop.conf.Configuration;
5250
import org.apache.hadoop.conf.Configured;
51+
import org.apache.hadoop.fs.s3a.impl.CSEMaterials;
5352
import org.apache.hadoop.fs.s3a.statistics.impl.AwsStatisticsCollector;
5453
import org.apache.hadoop.fs.store.LogExactlyOnce;
5554

@@ -99,39 +98,33 @@ public S3Client createS3Client(
9998

10099
@Override
101100
public S3Client createS3EncryptionClient(final S3AsyncClient s3AsyncClient,
102-
final S3Client s3Client, final KmsKeyring keyring) {
101+
final S3Client s3Client, final CSEMaterials cseMaterials) {
103102

104-
return S3EncryptionClient.builder()
105-
.wrappedAsyncClient(s3AsyncClient)
106-
.wrappedClient(s3Client)
107-
.enableLegacyUnauthenticatedModes(true)
108-
.keyring(keyring)
109-
.build();
103+
S3EncryptionClient.Builder s3EncryptionClientBuilder =
104+
S3EncryptionClient.builder().wrappedAsyncClient(s3AsyncClient).wrappedClient(s3Client)
105+
.enableLegacyUnauthenticatedModes(true);
106+
107+
if (cseMaterials.getCseKeyType().equals(CSEMaterials.CSEKeyType.KMS)) {
108+
s3EncryptionClientBuilder.kmsKeyId(cseMaterials.getKmsKeyId());
109+
}
110+
111+
return s3EncryptionClientBuilder.build();
110112
}
111113

112114

113115
@Override
114116
public S3AsyncClient createS3AsyncEncryptionClient(final S3AsyncClient s3AsyncClient,
115-
final KmsKeyring kmsKeyring) {
117+
final CSEMaterials cseMaterials) {
116118

117-
return S3AsyncEncryptionClient.builder()
118-
.wrappedClient(s3AsyncClient)
119-
.enableLegacyUnauthenticatedModes(true)
120-
.keyring(kmsKeyring)
121-
.build();
122-
}
119+
S3AsyncEncryptionClient.Builder s3EncryptionAsyncClientBuilder =
120+
S3AsyncEncryptionClient.builder().wrappedClient(s3AsyncClient)
121+
.enableLegacyUnauthenticatedModes(true);
123122

124-
public KmsKeyring createKmsKeyring(final S3ClientCreationParameters parameters,
125-
final String kmsKeyId) {
126-
// TODO: Figure out how to handle region here
127-
KmsClient kmsClient = KmsClient.builder()
128-
.credentialsProvider(parameters.getCredentialSet())
129-
.build();
123+
if (cseMaterials.getCseKeyType().equals(CSEMaterials.CSEKeyType.KMS)) {
124+
s3EncryptionAsyncClientBuilder.kmsKeyId(cseMaterials.getKmsKeyId());
125+
}
130126

131-
return KmsKeyring.builder()
132-
.kmsClient(kmsClient)
133-
.wrappingKeyId(kmsKeyId)
134-
.build();
127+
return s3EncryptionAsyncClientBuilder.build();
135128
}
136129

137130
@Override

‎hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@
126126
import org.apache.hadoop.fs.s3a.impl.AWSCannedACL;
127127
import org.apache.hadoop.fs.s3a.impl.AWSHeaders;
128128
import org.apache.hadoop.fs.s3a.impl.BulkDeleteRetryHandler;
129+
import org.apache.hadoop.fs.s3a.impl.CSEMaterials;
129130
import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy;
130131
import org.apache.hadoop.fs.s3a.impl.ContextAccessors;
131132
import org.apache.hadoop.fs.s3a.impl.CopyFromLocalOperation;
@@ -1004,9 +1005,14 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException {
10041005

10051006
if (isCSEEnabled) {
10061007
String kmsKeyId = getS3EncryptionKey(bucket, conf, true);
1007-
KmsKeyring kmsKeyring = clientFactory.createKmsKeyring(parameters, kmsKeyId);
1008-
s3Client = clientFactory.createS3EncryptionClient(s3AsyncClient, s3Client, kmsKeyring);
1009-
s3AsyncClient = clientFactory.createS3AsyncEncryptionClient(s3AsyncClient, kmsKeyring);
1008+
1009+
CSEMaterials cseMaterials = new CSEMaterials()
1010+
.withCSEKeyType(CSEMaterials.CSEKeyType.KMS)
1011+
.withKmsKeyId(kmsKeyId);
1012+
1013+
1014+
s3Client = clientFactory.createS3EncryptionClient(s3AsyncClient, s3Client, cseMaterials);
1015+
s3AsyncClient = clientFactory.createS3AsyncEncryptionClient(s3AsyncClient, cseMaterials);
10101016
}
10111017

10121018
transferManager = clientFactory.createS3TransferManager(getS3AsyncClient());

‎hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,10 @@
3232
import software.amazon.awssdk.services.s3.S3AsyncClient;
3333
import software.amazon.awssdk.services.s3.S3Client;
3434
import software.amazon.awssdk.transfer.s3.S3TransferManager;
35-
import software.amazon.encryption.s3.materials.KmsKeyring;
3635

3736
import org.apache.hadoop.classification.InterfaceAudience;
3837
import org.apache.hadoop.classification.InterfaceStability;
38+
import org.apache.hadoop.fs.s3a.impl.CSEMaterials;
3939
import org.apache.hadoop.fs.s3a.statistics.StatisticsFromAwsSdk;
4040

4141
import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_ENDPOINT;
@@ -90,31 +90,23 @@ S3AsyncClient createS3AsyncClient(URI uri,
9090
*
9191
* @param s3AsyncClient The asynchronous S3 client, will be used for cryptographic operations.
9292
* @param s3Client The synchronous S3 client, will be used for non cryptographic operations.
93-
* @param kmsKeyring kms wrapping key to be used
93+
* @param cseMaterials cse key and type to use
9494
* @return S3EncryptionClient
9595
*/
9696
S3Client createS3EncryptionClient(S3AsyncClient s3AsyncClient, S3Client s3Client,
97-
KmsKeyring kmsKeyring);
97+
CSEMaterials cseMaterials);
9898

9999

100100
/**
101101
* Creates a new {@link software.amazon.encryption.s3.S3AsyncEncryptionClient}.
102102
* Used when client side encryption is enabled.
103103
*
104104
* @param s3AsyncClient The asynchronous S3 client, will be used for cryptographic operations.
105-
* @param kmsKeyring kms wrapping key to be used
105+
* @param cseMaterials cse key and type to use
106106
* @return S3AsyncEncryptionClient
107107
*/
108-
S3AsyncClient createS3AsyncEncryptionClient(S3AsyncClient s3AsyncClient, KmsKeyring kmsKeyring);
108+
S3AsyncClient createS3AsyncEncryptionClient(S3AsyncClient s3AsyncClient, CSEMaterials cseMaterials);
109109

110-
/**
111-
* Creates KmsKeyring to be used by the encryption clients.
112-
*
113-
* @param parameters parameter object
114-
* @param kmsKeyId kms wrapping key to be used
115-
* @return KmsKeyring
116-
*/
117-
KmsKeyring createKmsKeyring(S3ClientCreationParameters parameters, String kmsKeyId);
118110

119111
/**
120112
* Creates a new {@link S3TransferManager}.

‎hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEMaterials.java‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.fs.s3a.impl;
20+
21+
/**
22+
* This class is for storing information about key type and corresponding key
23+
* to be used for client side encryption.
24+
*/
25+
public class CSEMaterials {
26+
27+
/**
28+
* Enum for CSE key types.
29+
*/
30+
public enum CSEKeyType {
31+
KMS,
32+
AES,
33+
RSA
34+
}
35+
36+
/**
37+
* The KMS key Id.
38+
*/
39+
private String kmsKeyId;
40+
41+
/**
42+
* The CSE key type to use.
43+
*/
44+
private CSEKeyType cseKeyType;
45+
46+
/**
47+
* Kms key id to use.
48+
* @param value new value
49+
* @return the builder
50+
*/
51+
public CSEMaterials withKmsKeyId(
52+
final String value) {
53+
kmsKeyId = value;
54+
return this;
55+
}
56+
57+
/**
58+
* Get the Kms key id to use.
59+
* @return the kms key id.
60+
*/
61+
public String getKmsKeyId() {
62+
return kmsKeyId;
63+
}
64+
65+
/**
66+
* CSE key type to use.
67+
* @param value new value
68+
* @return the builder
69+
*/
70+
public CSEMaterials withCSEKeyType(
71+
final CSEKeyType value) {
72+
cseKeyType = value;
73+
return this;
74+
}
75+
76+
/**
77+
* Get the CSE key type.
78+
* @return CSE key type
79+
*/
80+
public CSEKeyType getCseKeyType() {
81+
return cseKeyType;
82+
}
83+
84+
}

‎hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@
3131
import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest;
3232
import software.amazon.awssdk.services.s3.model.ListMultipartUploadsResponse;
3333
import software.amazon.awssdk.transfer.s3.S3TransferManager;
34-
import software.amazon.encryption.s3.materials.KmsKeyring;
34+
35+
import org.apache.hadoop.fs.s3a.impl.CSEMaterials;
3536

3637
/**
3738
* An {@link S3ClientFactory} that returns Mockito mocks of the {@link S3Client}
@@ -64,25 +65,18 @@ public S3AsyncClient createS3AsyncClient(URI uri, final S3ClientCreationParamete
6465

6566
@Override
6667
public S3Client createS3EncryptionClient(final S3AsyncClient s3AsyncClient,
67-
final S3Client s3Client, final KmsKeyring keyring) {
68+
final S3Client s3Client, final CSEMaterials cseMaterials) {
6869
S3Client s3 = mock(S3Client.class);
6970
return s3;
7071
}
7172

72-
7373
@Override
7474
public S3AsyncClient createS3AsyncEncryptionClient(final S3AsyncClient s3AsyncClient,
75-
final KmsKeyring kmsKeyring) {
75+
final CSEMaterials cseMaterials) {
7676
S3AsyncClient s3 = mock(S3AsyncClient.class);
7777
return s3;
7878
}
7979

80-
@Override
81-
public KmsKeyring createKmsKeyring(final S3ClientCreationParameters parameters,
82-
final String kmsKeyId) {
83-
KmsKeyring kmsKeyring = mock(KmsKeyring.class);
84-
return kmsKeyring;
85-
};
8680

8781
@Override
8882
public S3TransferManager createS3TransferManager(S3AsyncClient s3AsyncClient) {

0 commit comments

Comments
 (0)