Skip to content

Commit a1e5584

Browse files
rsotn-maprekrivokonmapr
rsotn-mapr
authored and
ekrivokonmapr
committed
MapR [SPARK-182] Spark Project External Kafka Producer v09 unit tests fixed (apache#263)
1 parent 8caa9da commit a1e5584

4 files changed

Lines changed: 310 additions & 0 deletions

File tree

‎external/kafka-producer/pom.xml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,5 +140,31 @@
140140
<build>
141141
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
142142
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
143+
<plugins>
144+
<!-- Surefire runs all Java tests -->
145+
<plugin>
146+
<groupId>org.apache.maven.plugins</groupId>
147+
<artifactId>maven-surefire-plugin</artifactId>
148+
<version>2.20.1</version>
149+
<!-- Note config is repeated in scalatest config -->
150+
<configuration>
151+
<systemProperties>
152+
<java.security.auth.login.config>${project.basedir}/src/test/resources/mapr.login.conf</java.security.auth.login.config>
153+
</systemProperties>
154+
</configuration>
155+
</plugin>
156+
<!-- Scalatest runs all Scala tests -->
157+
<plugin>
158+
<groupId>org.scalatest</groupId>
159+
<artifactId>scalatest-maven-plugin</artifactId>
160+
<version>1.0</version>
161+
<!-- Note config is repeated in surefire config -->
162+
<configuration>
163+
<systemProperties>
164+
<java.security.auth.login.config>${project.basedir}/src/test/resources/mapr.login.conf</java.security.auth.login.config>
165+
</systemProperties>
166+
</configuration>
167+
</plugin>
168+
</plugins>
143169
</build>
144170
</project>

‎external/kafka-producer/src/test/resources/mapr.login.conf‎

Lines changed: 281 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,281 @@
1+
/**
2+
* The KerberosBugWorkAroundLoginModule works around a Java 6 bug where it
3+
* doesn't pick up KRB5CCName properly. This is not needed with recent
4+
* patch levels of Java 7.
5+
*
6+
* Used by maprlogin and MapRLogin for client authentication
7+
*/
8+
MAPR_CLIENT_KERBEROS {
9+
org.apache.hadoop.security.login.KerberosBugWorkAroundLoginModule required
10+
useTicketCache=true
11+
doNotPrompt=true;
12+
};
13+
14+
/**
15+
* Used by CLDB for authenticating users.
16+
* The principal value is used as the Kerberos server identity of the CLDB.
17+
* This is generated by configure.sh by default. You can edit this value
18+
* but ensure that every CLDB node has the same principal name.
19+
*/
20+
MAPR_SERVER_KERBEROS {
21+
com.sun.security.auth.module.Krb5LoginModule required
22+
refreshKrb5Config=true
23+
doNotPrompt=true
24+
useKeyTab=true
25+
storeKey=true
26+
keyTab="/opt/mapr/conf/mapr.keytab"
27+
isInitiator=false
28+
principal="mapr/cyber.mapr.cluster";
29+
};
30+
31+
/**
32+
* Used by web servers for SPNEGO authentication. These include
33+
* MapR Webserver/MCS server in case SPNEGO REST APIs authentication is enabled
34+
* JT/TT/HBase/Oozie/etc web UIs configured to use
35+
org.apache.hadoop.security.authentication.server.MultiMechsAuthenticationHandler
36+
* The principal value is the kerberos server identity provided by the
37+
* web server for SPNEGO. Recall that SPNEGO identities are HTTP/hostname
38+
* as perceived by the client. The value is automatically generated by
39+
* configure.sh. However, if the hostname placed here is not the hostname
40+
* used by your clients, you may need to edit it.
41+
* Remember that each web server node will have a different value for
42+
* the principal based upon that node's hostname.
43+
*/
44+
MAPR_WEBSERVER_KERBEROS {
45+
com.sun.security.auth.module.Krb5LoginModule required
46+
refreshKrb5Config=true
47+
doNotPrompt=true
48+
useKeyTab=true
49+
storeKey=true
50+
keyTab="/opt/mapr/conf/mapr.keytab"
51+
isInitiator=false
52+
principal="HTTP/node1";
53+
};
54+
55+
/**
56+
* Used for password authentication with PAM. jpam is a Java wrapper
57+
* for PAM. The serviceName below determines which PAM configurations
58+
* are to be used for validating passwords. The list is used in the order
59+
* shown. A failure is ignored and the system proceeds to the next entry.
60+
* If your PAM configurations (typically in /etc/pam.d) are not the same
61+
* as our provided defaults, you may need to change the serviceName values,
62+
* add stanzas, or remove stanzas.
63+
*
64+
* mapr-admin is there by default as a placeholder should you choose to
65+
* create MapR specific PAM configuration. If you have no mapr-admin
66+
* PAM configuration, you can just remove it.
67+
*/
68+
jpamLogin {
69+
net.sf.jpam.jaas.JpamLoginModule Sufficient
70+
serviceName="sudo"
71+
debug=true;
72+
net.sf.jpam.jaas.JpamLoginModule Sufficient
73+
serviceName="sshd"
74+
debug=true;
75+
net.sf.jpam.jaas.JpamLoginModule Sufficient
76+
serviceName="mapr-admin"
77+
debug=true;
78+
};
79+
80+
/********************************************************************
81+
DO NOT EDIT BELOW THIS LINE WITHOUT CONTACTING MAPR SUPPORT
82+
**********************************************************************/
83+
84+
/**
85+
* Used by Zookeeper
86+
*/
87+
Server {
88+
com.mapr.security.maprsasl.MaprSecurityLoginModule required
89+
checkUGI=false
90+
cldbkeylocation="/opt/mapr/conf/cldb.key"
91+
debug=true;
92+
};
93+
94+
Client {
95+
com.mapr.security.maprsasl.MaprSecurityLoginModule required
96+
checkUGI=false
97+
authMech="MAPR-SECURITY"
98+
debug=true;
99+
};
100+
101+
/**
102+
* Used by Zookeeper - Security is off
103+
*/
104+
Server_simple {
105+
com.mapr.security.simplesasl.GenericLoginModule required
106+
debug=true;
107+
};
108+
109+
Client_simple {
110+
com.mapr.security.simplesasl.GenericLoginModule required
111+
authMech="SIMPLE-SECURITY";
112+
};
113+
114+
/**
115+
* used to obtain MapR credentials
116+
* TODO: rename from maprsasl to something else? maprauth?
117+
*/
118+
hadoop_maprsasl {
119+
org.apache.hadoop.security.login.GenericOSLoginModule required;
120+
com.mapr.security.maprsasl.MaprSecurityLoginModule required
121+
checkUGI=false;
122+
org.apache.hadoop.security.login.HadoopLoginModule required
123+
principalPriority=com.mapr.security.MapRPrincipal;
124+
};
125+
126+
hadoop_maprsasl_keytab {
127+
org.apache.hadoop.security.login.GenericOSLoginModule required;
128+
com.mapr.security.maprsasl.MaprSecurityLoginModule required
129+
checkUGI=false
130+
useServerKey=true;
131+
org.apache.hadoop.security.login.HadoopLoginModule required
132+
principalPriority=com.mapr.security.MapRPrincipal;
133+
};
134+
135+
hadoop_maprsasl_permissive {
136+
org.apache.hadoop.security.login.PermissiveLoginModule required;
137+
org.apache.hadoop.security.login.GenericOSLoginModule required;
138+
com.mapr.security.maprsasl.MaprSecurityLoginModule required
139+
checkUGI=false;
140+
org.apache.hadoop.security.login.HadoopLoginModule required
141+
principalPriority=com.mapr.security.MapRPrincipal;
142+
};
143+
144+
hadoop_maprsasl_permissive_keytab {
145+
org.apache.hadoop.security.login.PermissiveLoginModule required;
146+
org.apache.hadoop.security.login.GenericOSLoginModule required;
147+
com.mapr.security.maprsasl.MaprSecurityLoginModule required
148+
checkUGI=false
149+
useServerKey=true;
150+
org.apache.hadoop.security.login.HadoopLoginModule required
151+
principalPriority=com.mapr.security.MapRPrincipal;
152+
};
153+
154+
/**
155+
* intended for use with Kerberos authentication, no MapR
156+
*/
157+
hadoop_kerberos {
158+
org.apache.hadoop.security.login.KerberosBugWorkAroundLoginModule required
159+
useTicketCache=true
160+
renewTGT=true
161+
doNotPrompt=true;
162+
org.apache.hadoop.security.login.GenericOSLoginModule required;
163+
org.apache.hadoop.security.login.HadoopLoginModule required;
164+
};
165+
166+
/**
167+
* TODO:
168+
* left out isInitial
169+
* should I leave out renewTGT?
170+
*/
171+
hadoop_kerberos_keytab {
172+
org.apache.hadoop.security.login.KerberosBugWorkAroundLoginModule required
173+
refreshKrb5Config=true
174+
doNotPrompt=true
175+
useKeyTab=true
176+
storeKey=true;
177+
org.apache.hadoop.security.login.GenericOSLoginModule required;
178+
org.apache.hadoop.security.login.HadoopLoginModule required;
179+
};
180+
181+
/**
182+
* TODO: KerberosPrincipal preferred, should be add clear to principalPriority?
183+
* authenticate using hybrid of kerberos and MapR
184+
* maprticket must already exist on file system as MapR login module
185+
* cannot get kerberos identity from subject for implicit login.
186+
*/
187+
hadoop_hybrid {
188+
org.apache.hadoop.security.login.KerberosBugWorkAroundLoginModule optional
189+
useTicketCache=true
190+
renewTGT=true
191+
doNotPrompt=true;
192+
com.mapr.security.maprsasl.MaprSecurityLoginModule required
193+
checkUGI=false;
194+
org.apache.hadoop.security.login.GenericOSLoginModule required;
195+
org.apache.hadoop.security.login.HadoopLoginModule required
196+
principalPriority=com.mapr.security.MapRPrincipal;
197+
};
198+
199+
hadoop_hybrid_keytab {
200+
com.mapr.security.maprsasl.MaprSecurityLoginModule required
201+
checkUGI=false
202+
useServerKey=true;
203+
org.apache.hadoop.security.login.KerberosBugWorkAroundLoginModule optional
204+
refreshKrb5Config=true
205+
doNotPrompt=true
206+
useKeyTab=true
207+
storeKey=true;
208+
org.apache.hadoop.security.login.GenericOSLoginModule required;
209+
org.apache.hadoop.security.login.HadoopLoginModule required
210+
principalPriority=com.mapr.security.MapRPrincipal;
211+
};
212+
213+
hadoop_hybrid_permissive {
214+
org.apache.hadoop.security.login.KerberosBugWorkAroundLoginModule optional
215+
useTicketCache=true
216+
renewTGT=true
217+
doNotPrompt=true;
218+
com.mapr.security.maprsasl.MaprSecurityLoginModule required
219+
checkUGI=false;
220+
org.apache.hadoop.security.login.GenericOSLoginModule required;
221+
org.apache.hadoop.security.login.PermissiveLoginModule required;
222+
org.apache.hadoop.security.login.HadoopLoginModule required
223+
principalPriority=com.mapr.security.MapRPrincipal;
224+
};
225+
226+
hadoop_hybrid_permissive_keytab {
227+
org.apache.hadoop.security.login.KerberosBugWorkAroundLoginModule optional
228+
refreshKrb5Config=true
229+
doNotPrompt=true
230+
useKeyTab=true
231+
storeKey=true;
232+
com.mapr.security.maprsasl.MaprSecurityLoginModule required
233+
checkUGI=false
234+
useServerKey=true;
235+
org.apache.hadoop.security.login.GenericOSLoginModule required;
236+
org.apache.hadoop.security.login.PermissiveLoginModule required;
237+
org.apache.hadoop.security.login.HadoopLoginModule required
238+
principalPriority=com.mapr.security.MapRPrincipal;
239+
};
240+
241+
/**
242+
* simple login, just get OS creds
243+
*/
244+
hadoop_simple {
245+
org.apache.hadoop.security.login.GenericOSLoginModule required;
246+
org.apache.hadoop.security.login.HadoopLoginModule required;
247+
};
248+
/* all configurations should have corresponding a "_keytab" section for
249+
* loginFromKeytab(), even if it duplicates the one without.
250+
*/
251+
hadoop_simple_keytab {
252+
org.apache.hadoop.security.login.GenericOSLoginModule required;
253+
org.apache.hadoop.security.login.HadoopLoginModule required;
254+
};
255+
256+
/**
257+
* these are used when there is no jvm or config setting for hadoop.login
258+
* it's your backup basically. No MapR components should depend on this
259+
* but it's handy for dealing with unfamilar code.
260+
* Note the intentional error to make sure you don't use by accident
261+
*/
262+
hadoop_default {
263+
org.apache.hadoop.security.login.GenericOSLoginModule required;
264+
com.mapr.security.maprsasl.MaprSecurityLoginModule required
265+
checkUGI=false;
266+
org.apache.hadoop.security.login.HadoopLoginModule required
267+
principalPriority=com.mapr.security.MapRPrincipal;
268+
};
269+
270+
/**
271+
* keytab version of previous
272+
*/
273+
hadoop_default_keytab {
274+
YouShouldntSeeThisErrorUnlessYourJVMhadoop.loginPropertiesAreBad required;
275+
org.apache.hadoop.security.login.GenericOSLoginModule required;
276+
com.mapr.security.maprsasl.MaprSecurityLoginModule required
277+
checkUGI=false
278+
useServerKey=true;
279+
org.apache.hadoop.security.login.HadoopLoginModule required
280+
principalPriority=com.mapr.security.MapRPrincipal;
281+
};

‎external/kafka-producer/src/test/scala/org/apache/spark/streaming/kafka/producer/RDDProducerSuite.scala‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ class RDDProducerSuite extends BaseKafkaProducerTest {
3434
private val recordValue = "value"
3535
private val partition = 0
3636
private val testConf = new ProducerConf(bootstrapServers = List("localhost:9092"))
37+
.withKeySerializer("org.apache.kafka.common.serialization.StringSerializer")
38+
3739
private var sparkContext: SparkContext = _
3840

3941
before {

‎external/kafka-producer/src/test/scala/org/apache/spark/streaming/kafka/producer/utils/EmbeddedKafka.scala‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ private[spark] final class EmbeddedKafka(
4545
"broker.id" -> "1",
4646
"zookeeper.connect" -> zkConnect,
4747
"host.name" -> "localhost",
48+
"offsets.topic.replication.factor" -> "1",
4849
"log.dir" -> logDir.toString)
4950

5051
val props = kafkaProps.getOrElse(new Properties())

0 commit comments

Comments
 (0)