Skip to content

Commit 8b67a90

Browse files
JohnZZGithubJohnZZGithub
committed
YARN-2774. support secure clusters in shared cache manager
1 parent 460ba7f commit 8b67a90

15 files changed

Lines changed: 368 additions & 12 deletions

File tree

‎hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ It's recommended to have them share a Unix group, e.g. `hadoop`. See also "[Mapp
4242
| User:Group | Daemons |
4343
|:--------------|:----------------------------------------------------|
4444
| hdfs:hadoop | NameNode, Secondary NameNode, JournalNode, DataNode |
45-
| yarn:hadoop | ResourceManager, NodeManager |
45+
| yarn:hadoop | ResourceManager, NodeManager, SharedCacheManager |
4646
| mapred:hadoop | MapReduce JobHistory Server |
4747

4848
### Kerberos principals for Hadoop Daemons
@@ -117,6 +117,18 @@ The NodeManager keytab file, on each host, should look like the following:
117117
4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
118118
4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
119119

120+
The SharedCacheManager keytab file, on that host, should look like the following:
121+
122+
$ klist -e -k -t /etc/security/keytab/scm.service.keytab
123+
Keytab name: FILE:/etc/security/keytab/scm.service.keytab
124+
KVNO Timestamp Principal
125+
4 07/18/11 21:08:09 scm/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
126+
4 07/18/11 21:08:09 scm/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
127+
4 07/18/11 21:08:09 scm/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
128+
4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
129+
4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
130+
4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
131+
120132
#### MapReduce JobHistory Server
121133

122134
The MapReduce JobHistory Server keytab file, on that host, should look like the following:
@@ -333,6 +345,13 @@ The following settings allow configuring SSL access to the NameNode web UI (opti
333345
| `yarn.nodemanager.linux-container-executor.path` | `/path/to/bin/container-executor` | The path to the executable of Linux container executor. |
334346
| `yarn.nodemanager.webapp.https.address` | `0.0.0.0:8044` | The https adddress of the NM web application. |
335347

348+
### SharedCacheManager
349+
350+
| Parameter | Value | Notes |
351+
|:-----------------------------|:------------------------------------------|:----------------------------------------------------|
352+
| `yarn.sharedcache.principal` | `scm/_HOST@REALM.TLD` | Kerberos principal name for the SharedCacheManager. |
353+
| `yarn.sharedcache.keytab` | `/etc/security/keytab/scm.service.keytab` | Kerberos keytab file for the SharedCacheManager. |
354+
336355
### Configuration for WebAppProxy
337356

338357
The `WebAppProxy` provides a proxy between the web applications exported by an application and an end user. If security is enabled it will warn users before accessing a potentially unsafe web application. Authentication and authorization using the proxy is handled just like any other privileged web application.

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientSCMProtocolPB.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,16 @@
1818
package org.apache.hadoop.yarn.api;
1919

2020
import org.apache.hadoop.ipc.ProtocolInfo;
21+
import org.apache.hadoop.security.KerberosInfo;
22+
import org.apache.hadoop.yarn.conf.YarnConfiguration;
2123
import org.apache.hadoop.yarn.proto.ClientSCMProtocol.ClientSCMProtocolService;
2224

25+
/**
26+
* This is protocol interface used by shared cache client to interacte with
27+
* shared cache manager.
28+
*/
29+
@KerberosInfo(
30+
serverPrincipal = YarnConfiguration.SCM_PRINCIPAL)
2331
@ProtocolInfo(protocolName = "org.apache.hadoop.yarn.api.ClientSCMProtocolPB",
2432
protocolVersion = 1)
2533
public interface ClientSCMProtocolPB extends

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2425,6 +2425,18 @@ public static boolean isAclEnabled(Configuration conf) {
24252425
YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_NODEMANAGER_PROTOCOL =
24262426
"security.applicationmaster-nodemanager.applicationmaster.protocol.acl";
24272427

2428+
public static final String
2429+
YARN_SECURITY_SERVICE_AUTHORIZATION_SHAREDCACHEMANAGER_CLIENT_PROTOCOL =
2430+
"security.sharedcachemanager.client.protocol.acl";
2431+
2432+
public static final String
2433+
YARN_SECURITY_SERVICE_AUTHORIZATION_SHAREDCACHEMANAGER_ADMIN_PROTOCOL =
2434+
"security.sharedcachemanager.admin.protocol.acl";
2435+
2436+
public static final String
2437+
YARN_SECURITY_SERVICE_AUTHORIZATION_SHAREDCACHEMANAGER_UPLOADER_PROTOCOL =
2438+
"security.sharedcachemanager.uploader.protocol.acl";
2439+
24282440
/** No. of milliseconds to wait between sending a SIGTERM and SIGKILL
24292441
* to a running container */
24302442
public static final String NM_SLEEP_DELAY_BEFORE_SIGKILL_MS =
@@ -3342,6 +3354,13 @@ public static boolean isAclEnabled(Configuration conf) {
33423354
SHARED_CACHE_PREFIX + "nm.uploader.thread-count";
33433355
public static final int DEFAULT_SHARED_CACHE_NM_UPLOADER_THREAD_COUNT = 20;
33443356

3357+
/** The keytab for the shared cache manager.*/
3358+
public static final String SCM_KEYTAB =
3359+
SHARED_CACHE_PREFIX + "keytab";
3360+
3361+
public static final String SCM_PRINCIPAL =
3362+
SHARED_CACHE_PREFIX + "principal";
3363+
33453364
////////////////////////////////
33463365
// Federation Configs
33473366
////////////////////////////////

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/SCMAdminProtocolPB.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,14 @@
2020
import org.apache.hadoop.classification.InterfaceAudience.Private;
2121
import org.apache.hadoop.classification.InterfaceStability.Unstable;
2222
import org.apache.hadoop.ipc.ProtocolInfo;
23+
import org.apache.hadoop.security.KerberosInfo;
24+
import org.apache.hadoop.yarn.conf.YarnConfiguration;
2325
import org.apache.hadoop.yarn.proto.SCMAdminProtocol.SCMAdminProtocolService;
2426

2527
@Private
2628
@Unstable
29+
@KerberosInfo(
30+
serverPrincipal = YarnConfiguration.SCM_PRINCIPAL)
2731
@ProtocolInfo(protocolName = "org.apache.hadoop.yarn.server.api.SCMAdminProtocolPB",
2832
protocolVersion = 1)
2933
public interface SCMAdminProtocolPB extends

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2836,6 +2836,45 @@
28362836
<value>20</value>
28372837
</property>
28382838

2839+
<property>
2840+
<description>The Kerberos principal for the shared cache manager.
2841+
</description>
2842+
<name>yarn.sharedcache.principal</name>
2843+
<value></value>
2844+
</property>
2845+
2846+
<property>
2847+
<description>The Kerberos keytab for the shared cache manager.
2848+
</description>
2849+
<name>yarn.sharedcache.keytab</name>
2850+
<value></value>
2851+
</property>
2852+
2853+
<property>
2854+
<description>
2855+
ACL protocol used in shared cache manager to control client request.
2856+
</description>
2857+
<name>security.sharedcachemanager.client.protocol.acl</name>
2858+
<value></value>
2859+
</property>
2860+
2861+
<property>
2862+
<description>
2863+
ACL protocol used in shared cache manager for admin RPC request.
2864+
</description>
2865+
<name>security.sharedcachemanager.admin.protocol.acl</name>
2866+
<value></value>
2867+
</property>
2868+
2869+
<property>
2870+
<description>
2871+
ACL protocol used in shared cache manager for uploader requestion from
2872+
node manager.
2873+
</description>
2874+
<name>security.sharedcachemanager.uploader.protocol.acl</name>
2875+
<value></value>
2876+
</property>
2877+
28392878
<property>
28402879
<description>
28412880
ACL protocol for use in the Timeline server.

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/SCMUploaderProtocolPB.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,16 @@
1818
package org.apache.hadoop.yarn.server.api;
1919

2020
import org.apache.hadoop.ipc.ProtocolInfo;
21+
import org.apache.hadoop.security.KerberosInfo;
22+
import org.apache.hadoop.yarn.conf.YarnConfiguration;
2123
import org.apache.hadoop.yarn.proto.SCMUploaderProtocol.SCMUploaderProtocolService;
2224

25+
/**
26+
* Protocol interface that provide uploading interface. The client should be
27+
* node manager and the server is shared cache manager.
28+
*/
29+
@KerberosInfo(
30+
serverPrincipal = YarnConfiguration.SCM_PRINCIPAL)
2331
@ProtocolInfo(protocolName = "org.apache.hadoop.yarn.server.api.SCMUploaderProtocolPB",
2432
protocolVersion = 1)
2533
public interface SCMUploaderProtocolPB extends

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/ClientProtocolService.java‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,15 @@
2121
import java.io.IOException;
2222
import java.net.InetSocketAddress;
2323

24+
import com.google.common.annotations.VisibleForTesting;
2425
import org.apache.hadoop.classification.InterfaceAudience.Private;
2526
import org.apache.hadoop.classification.InterfaceStability.Evolving;
2627
import org.apache.hadoop.conf.Configuration;
28+
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
2729
import org.apache.hadoop.fs.Path;
2830
import org.apache.hadoop.ipc.Server;
2931
import org.apache.hadoop.security.UserGroupInformation;
32+
import org.apache.hadoop.security.authorize.PolicyProvider;
3033
import org.apache.hadoop.service.AbstractService;
3134
import org.apache.hadoop.yarn.api.ClientSCMProtocol;
3235
import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest;
@@ -41,6 +44,7 @@
4144
import org.apache.hadoop.yarn.ipc.YarnRPC;
4245
import org.apache.hadoop.yarn.server.sharedcache.SharedCacheUtil;
4346
import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.ClientSCMMetrics;
47+
import org.apache.hadoop.yarn.server.sharedcachemanager.security.SCMPolicyProvider;
4448
import org.apache.hadoop.yarn.server.sharedcachemanager.store.SCMStore;
4549
import org.apache.hadoop.yarn.server.sharedcachemanager.store.SharedCacheResourceReference;
4650
import org.slf4j.Logger;
@@ -105,7 +109,13 @@ protected void serviceStart() throws Exception {
105109
conf.getInt(YarnConfiguration.SCM_CLIENT_SERVER_THREAD_COUNT,
106110
YarnConfiguration.DEFAULT_SCM_CLIENT_SERVER_THREAD_COUNT));
107111

108-
// TODO (YARN-2774): Enable service authorization
112+
// TODO: dynamically load ACLs
113+
// Enable service authorization
114+
if (conf.getBoolean(
115+
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
116+
false)) {
117+
refreshServiceAcls(conf, SCMPolicyProvider.getInstance());
118+
}
109119

110120
this.server.start();
111121
clientBindAddress =
@@ -115,6 +125,12 @@ protected void serviceStart() throws Exception {
115125
super.serviceStart();
116126
}
117127

128+
private void refreshServiceAcls(Configuration configuration,
129+
PolicyProvider policyProvider) {
130+
this.server.refreshServiceAclWithLoadedConfiguration(configuration,
131+
policyProvider);
132+
}
133+
118134
@Override
119135
protected void serviceStop() throws Exception {
120136
if (this.server != null) {
@@ -190,4 +206,9 @@ private String getCacheEntryFilePath(String checksum, String filename) {
190206
return SharedCacheUtil.getCacheEntryPath(this.cacheDepth,
191207
this.cacheRoot, checksum) + Path.SEPARATOR_CHAR + filename;
192208
}
209+
210+
@VisibleForTesting
211+
protected Server getServer() {
212+
return server;
213+
}
193214
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SCMAdminProtocolService.java‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,15 @@
2121
import java.io.IOException;
2222
import java.net.InetSocketAddress;
2323

24+
import com.google.common.annotations.VisibleForTesting;
2425
import org.apache.hadoop.classification.InterfaceAudience.Private;
2526
import org.apache.hadoop.classification.InterfaceStability.Unstable;
2627
import org.apache.hadoop.conf.Configuration;
28+
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
2729
import org.apache.hadoop.ipc.Server;
2830
import org.apache.hadoop.security.AccessControlException;
2931
import org.apache.hadoop.security.UserGroupInformation;
32+
import org.apache.hadoop.security.authorize.PolicyProvider;
3033
import org.apache.hadoop.service.AbstractService;
3134
import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
3235
import org.apache.hadoop.yarn.server.api.SCMAdminProtocol;
@@ -38,6 +41,7 @@
3841
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
3942
import org.apache.hadoop.yarn.ipc.RPCUtil;
4043
import org.apache.hadoop.yarn.ipc.YarnRPC;
44+
import org.apache.hadoop.yarn.server.sharedcachemanager.security.SCMPolicyProvider;
4145
import org.slf4j.Logger;
4246
import org.slf4j.LoggerFactory;
4347

@@ -89,7 +93,14 @@ protected void serviceStart() throws Exception {
8993
conf.getInt(YarnConfiguration.SCM_ADMIN_CLIENT_THREAD_COUNT,
9094
YarnConfiguration.DEFAULT_SCM_ADMIN_CLIENT_THREAD_COUNT));
9195

92-
// TODO: Enable service authorization (see YARN-2774)
96+
// TODO: dynamically load ACLs
97+
// Enable service authorization
98+
if (conf.getBoolean(
99+
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
100+
false)) {
101+
refreshServiceAcls(
102+
conf, SCMPolicyProvider.getInstance());
103+
}
93104

94105
this.server.start();
95106
clientBindAddress =
@@ -99,6 +110,12 @@ protected void serviceStart() throws Exception {
99110
super.serviceStart();
100111
}
101112

113+
private void refreshServiceAcls(Configuration configuration,
114+
PolicyProvider policyProvider) {
115+
this.server.refreshServiceAclWithLoadedConfiguration(configuration,
116+
policyProvider);
117+
}
118+
102119
@Override
103120
protected void serviceStop() throws Exception {
104121
if (this.server != null) {
@@ -141,4 +158,9 @@ public RunSharedCacheCleanerTaskResponse runCleanerTask(
141158
response.setAccepted(true);
142159
return response;
143160
}
161+
162+
@VisibleForTesting
163+
public Server getServer() {
164+
return server;
165+
}
144166
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SharedCacheManager.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,15 @@
1818

1919
package org.apache.hadoop.yarn.server.sharedcachemanager;
2020

21+
import java.io.IOException;
22+
import java.net.InetSocketAddress;
23+
2124
import org.apache.hadoop.classification.InterfaceAudience.Private;
2225
import org.apache.hadoop.classification.InterfaceStability.Unstable;
2326
import org.apache.hadoop.conf.Configuration;
2427
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
2528
import org.apache.hadoop.metrics2.source.JvmMetrics;
29+
import org.apache.hadoop.security.SecurityUtil;
2630
import org.apache.hadoop.service.CompositeService;
2731
import org.apache.hadoop.util.ReflectionUtils;
2832
import org.apache.hadoop.util.ShutdownHookManager;
@@ -63,6 +67,13 @@ public SharedCacheManager() {
6367
@Override
6468
protected void serviceInit(Configuration conf) throws Exception {
6569

70+
try {
71+
doSecureLogin(conf);
72+
} catch(IOException ie) {
73+
throw new YarnRuntimeException(
74+
"Shared cache manager failed to login", ie);
75+
}
76+
6677
this.store = createSCMStoreService(conf);
6778
addService(store);
6879

@@ -130,6 +141,15 @@ private SCMWebServer createSCMWebServer(SharedCacheManager scm) {
130141
return new SCMWebServer(scm);
131142
}
132143

144+
protected void doSecureLogin(Configuration conf) throws IOException {
145+
InetSocketAddress socAddr = conf.getSocketAddr(
146+
YarnConfiguration.SCM_ADMIN_ADDRESS,
147+
YarnConfiguration.DEFAULT_SCM_ADMIN_ADDRESS,
148+
YarnConfiguration.DEFAULT_SCM_ADMIN_PORT);
149+
SecurityUtil.login(conf, YarnConfiguration.SCM_KEYTAB,
150+
YarnConfiguration.SCM_PRINCIPAL, socAddr.getHostName());
151+
}
152+
133153
@Override
134154
protected void serviceStop() throws Exception {
135155

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SharedCacheUploaderService.java‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,11 @@
2121
import java.io.IOException;
2222
import java.net.InetSocketAddress;
2323

24+
import com.google.common.annotations.VisibleForTesting;
2425
import org.apache.hadoop.conf.Configuration;
26+
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
2527
import org.apache.hadoop.ipc.Server;
28+
import org.apache.hadoop.security.authorize.PolicyProvider;
2629
import org.apache.hadoop.service.AbstractService;
2730
import org.apache.hadoop.yarn.conf.YarnConfiguration;
2831
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -35,6 +38,7 @@
3538
import org.apache.hadoop.yarn.server.api.protocolrecords.SCMUploaderNotifyRequest;
3639
import org.apache.hadoop.yarn.server.api.protocolrecords.SCMUploaderNotifyResponse;
3740
import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.SharedCacheUploaderMetrics;
41+
import org.apache.hadoop.yarn.server.sharedcachemanager.security.SCMPolicyProvider;
3842
import org.apache.hadoop.yarn.server.sharedcachemanager.store.SCMStore;
3943

4044
/**
@@ -81,7 +85,14 @@ protected void serviceStart() throws Exception {
8185
conf.getInt(YarnConfiguration.SCM_UPLOADER_SERVER_THREAD_COUNT,
8286
YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_THREAD_COUNT));
8387

84-
// TODO (YARN-2774): Enable service authorization
88+
// TODO: dynamically load ACLs
89+
// Enable service authorization
90+
if (conf.getBoolean(
91+
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
92+
false)) {
93+
refreshServiceAcls(
94+
conf, SCMPolicyProvider.getInstance());
95+
}
8596

8697
this.server.start();
8798
bindAddress =
@@ -91,6 +102,12 @@ protected void serviceStart() throws Exception {
91102
super.serviceStart();
92103
}
93104

105+
private void refreshServiceAcls(Configuration configuration,
106+
PolicyProvider policyProvider) {
107+
this.server.refreshServiceAclWithLoadedConfiguration(configuration,
108+
policyProvider);
109+
}
110+
94111
@Override
95112
protected void serviceStop() throws Exception {
96113
if (this.server != null) {
@@ -137,4 +154,9 @@ public SCMUploaderCanUploadResponse canUpload(
137154
response.setUploadable(true);
138155
return response;
139156
}
157+
158+
@VisibleForTesting
159+
protected Server getServer() {
160+
return server;
161+
}
140162
}

0 commit comments

Comments
 (0)