Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
Original file line number Diff line number Diff line change
Expand Up @@ -48,5 +48,4 @@ public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse r
}
return supportedTypes;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -77,5 +77,4 @@ public void testLoaderOverrideInvalidVersion() {
Assert.assertNotNull(shim);
Assert.assertEquals(DefaultHadoopShim.class, shim.getClass());
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse r

@Override
public FinalApplicationStatus applyFinalApplicationStatusCorrection(FinalApplicationStatus orig,
boolean isSessionMode, boolean isError) {
boolean isSessionMode, boolean isError) {
switch (orig) {
case FAILED:
// App is failed if dag failed in non-session mode or there was an error.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,5 +76,4 @@ public void testLoaderOverrideInvalidVersion() {
Assert.assertNotNull(shim);
Assert.assertEquals(DefaultHadoopShim.class, shim.getClass());
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ public abstract class HadoopShim {

/**
* Set up Hadoop Caller Context
*
* @param context Context to be set
*/
public void setHadoopCallerContext(String context) {
Expand All @@ -49,6 +50,7 @@ public void clearHadoopCallerContext() {

/**
* Extract supported Resource types from the RM's response when the AM registers
*
* @param response ApplicationMasterResponse from RM after registering
* @return Set of Resource types that are supported
*/
Expand All @@ -57,7 +59,7 @@ public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse r
}

public FinalApplicationStatus applyFinalApplicationStatusCorrection(FinalApplicationStatus orig,
boolean isSessionMode, boolean isError) {
boolean isSessionMode, boolean isError) {
return orig;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,12 @@ public abstract class HadoopShimProvider {

/**
* Provides a HadoopShim if the version matches the implementation of the Shim
*
* @param hadoopVersion Full version string of hadoop-common's version
* @param majorVersion Major version
* @param minorVersion Minor version
* @param majorVersion Major version
* @param minorVersion Minor version
* @return HadoopShim object if there is a match else null
*/
public abstract HadoopShim createHadoopShim(String hadoopVersion, int majorVersion,
int minorVersion);

int minorVersion);
}
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ public HadoopShimsLoader(Configuration conf) {
try {
LOG.debug("Using Reflection to create HadoopShim from provider class="
+ overrideProviderClassStr);
Class<HadoopShimProvider> clazz = (Class<HadoopShimProvider>)Class.forName(
Class<HadoopShimProvider> clazz = (Class<HadoopShimProvider>) Class.forName(
overrideProviderClassStr, true, Thread.currentThread().getContextClassLoader());
selectedShimProvider = clazz.newInstance();
selectedShim = selectedShimProvider.createHadoopShim(versionStr,
Expand All @@ -143,7 +143,6 @@ public HadoopShimsLoader(Configuration conf) {
+ ", hadoopVersion=" + versionStr
+ ", majorVersion=" + version.majorVersion
+ ", minorVersion=" + version.minorVersion);

}

public HadoopShim getHadoopShim() {
Expand Down Expand Up @@ -181,5 +180,4 @@ public Version(String versionString) {
}
}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -50,5 +50,4 @@ public void testInvalidOverride() {
HadoopShimsLoader loader = new HadoopShimsLoader(conf, true);
HadoopShim shim = loader.getHadoopShim();
}

}
10 changes: 5 additions & 5 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,8 @@
<scm.url>scm:git:https://gitbox.apache.org/repos/asf/tez.git</scm.url>
<frontend-maven-plugin.version>1.4</frontend-maven-plugin.version>
<findbugs-maven-plugin.version>3.0.5</findbugs-maven-plugin.version>
<maven-checkstyle-plugin.version>3.1.1</maven-checkstyle-plugin.version>
<checkstyle.version>8.35</checkstyle.version>
<maven-checkstyle-plugin.version>3.1.2</maven-checkstyle-plugin.version>
<checkstyle.version>9.3</checkstyle.version>
<dependency-check-maven.version>1.3.6</dependency-check-maven.version>
<test.build.data>${project.build.directory}/tmp</test.build.data>
</properties>
Expand Down Expand Up @@ -907,10 +907,10 @@
</dependency>
</dependencies>
<configuration>
<configLocation>checkstyle/checkstyle.xml</configLocation>
<suppressionsLocation>checkstyle/suppressions.xml</suppressionsLocation>
<configLocation>tez-build-tools/src/main/resources/checkstyle/checkstyle.xml</configLocation>
<suppressionsLocation>tez-build-tools/src/main/resources/checkstyle/suppressions.xml</suppressionsLocation>
<includeTestSourceDirectory>true</includeTestSourceDirectory>
<failOnViolation>false</failOnViolation>
<failOnViolation>true</failOnViolation>
<format>xml</format>
<format>html</format>
<outputFile>${project.build.directory}/test/checkstyle-errors.xml</outputFile>
Expand Down
20 changes: 9 additions & 11 deletions tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Expand Down Expand Up @@ -36,29 +36,28 @@ class AMConfiguration {
private LocalResource binaryConfLRsrc;

AMConfiguration(TezConfiguration tezConf, Map<String, LocalResource> localResources,
Credentials credentials) {
Credentials credentials) {
this.tezConf = tezConf;
if (localResources != null) {
addAMLocalResources(localResources);
}
if (credentials != null) {
setCredentials(credentials);
}

}

void addAMLocalResources(Map<String, LocalResource> localResources) {
this.amLocalResources.putAll(localResources);
}

void clearAMLocalResources() {
this.amLocalResources.clear();
}

void setCredentials(Credentials credentials) {
this.credentials = credentials;
}

void setTezConfiguration(TezConfiguration tezConf) {
this.tezConf = tezConf;
}
Expand All @@ -78,13 +77,12 @@ TezConfiguration getTezConfiguration() {
Credentials getCredentials() {
return credentials;
}

void setBinaryConfLR(LocalResource binaryConfLRsrc) {
this.binaryConfLRsrc = binaryConfLRsrc;
}

LocalResource getBinaryConfLR() {
return binaryConfLRsrc;
}

}
33 changes: 16 additions & 17 deletions tez-api/src/main/java/org/apache/tez/client/CallerContext.java
Original file line number Diff line number Diff line change
Expand Up @@ -64,38 +64,39 @@ private CallerContext() {

/**
* Instantiate the Caller Context
* @param context Context in which Tez is being invoked. For example, HIVE or PIG.
* @param callerId Caller ID. An ID to uniquely identifier the caller within the callerType
* namespace
*
* @param context Context in which Tez is being invoked. For example, HIVE or PIG.
* @param callerId Caller ID. An ID to uniquely identifier the caller within the callerType
* namespace
* @param callerType Type of the caller. Should ideally be used along with callerId to uniquely
* identify the caller. When used with YARN Timeline, this should map to
* the Timeline Entity Type. For example, HIVE_QUERY_ID.
* @param blob Free-form text or a json-representation of relevant meta-data.
* This can be used to describe the work being done. For example, for Hive,
* this could be the Hive query text.
* @param blob Free-form text or a json-representation of relevant meta-data.
* This can be used to describe the work being done. For example, for Hive,
* this could be the Hive query text.
* @return CallerContext
*/
public static CallerContext create(String context, String callerId,
String callerType, @Nullable String blob) {
String callerType, @Nullable String blob) {
return new CallerContext(context, callerId, callerType, blob);
}

/**
* Instantiate the Caller Context
*
* @param context Context in which Tez is being invoked. For example, HIVE or PIG.
* @param blob Free-form text or a json-representation of relevant meta-data.
* This can be used to describe the work being done. For example, for Hive,
* this could be the Hive query text.
* @param blob Free-form text or a json-representation of relevant meta-data.
* This can be used to describe the work being done. For example, for Hive,
* this could be the Hive query text.
* @return CallerContext
*/
@Private
public static CallerContext create(String context, @Nullable String blob) {
return new CallerContext(context, blob);
}


private CallerContext(String context, String callerId, String callerType,
@Nullable String blob) {
@Nullable String blob) {
if (callerId != null || callerType != null) {
setCallerIdAndType(callerId, callerType);
}
Expand Down Expand Up @@ -135,15 +136,15 @@ public CallerContext setContext(String context) {
}

/**
* @param callerId Caller ID. An ID to uniquely identifier the caller within the callerType
* namespace
* @param callerId Caller ID. An ID to uniquely identifier the caller within the callerType
* namespace
* @param callerType Type of the caller. Should ideally be used along with callerId to uniquely
* identify the caller. When used with YARN Timeline, this should map to
* the Timeline Entity Type. For example, HIVE_QUERY_ID.
*/
public CallerContext setCallerIdAndType(String callerId, String callerType) {
Preconditions.checkArgument(callerType != null && !callerType.isEmpty()
&& callerId != null && !callerId.isEmpty(),
&& callerId != null && !callerId.isEmpty(),
"Caller Id and Caller Type cannot be null or empty");
this.callerType = callerType;
this.callerId = callerId;
Expand Down Expand Up @@ -175,6 +176,4 @@ public String contextAsSimpleString() {
+ ", callerId=" + callerId
+ " }";
}


}
16 changes: 8 additions & 8 deletions tez-api/src/main/java/org/apache/tez/client/FrameworkClient.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Expand Down Expand Up @@ -97,7 +97,7 @@ public abstract ApplicationId submitApplication(ApplicationSubmissionContext app
public abstract boolean isRunning() throws IOException;

public TezAppMasterStatus getAMStatus(Configuration conf, ApplicationId appId,
UserGroupInformation ugi) throws TezException, ServiceException, IOException {
UserGroupInformation ugi) throws TezException, ServiceException, IOException {
DAGClientAMProtocolBlockingPB proxy = getProxy(conf, appId, ugi);

if (proxy == null) {
Expand All @@ -109,7 +109,7 @@ public TezAppMasterStatus getAMStatus(Configuration conf, ApplicationId appId,
}

public DAGClient submitDag(DAG dag, SubmitDAGRequestProto request, String clientName,
ApplicationId sessionAppId, long clientTimeout, UserGroupInformation ugi, TezConfiguration tezConf)
ApplicationId sessionAppId, long clientTimeout, UserGroupInformation ugi, TezConfiguration tezConf)
throws IOException, TezException, DAGSubmissionTimedOut {
DAGClientAMProtocolBlockingPB proxy = null;
try {
Expand Down Expand Up @@ -149,7 +149,7 @@ public DAGClient submitDag(DAG dag, SubmitDAGRequestProto request, String client
}

protected DAGClientAMProtocolBlockingPB waitForProxy(long clientTimeout, Configuration conf,
ApplicationId sessionAppId, UserGroupInformation ugi)
ApplicationId sessionAppId, UserGroupInformation ugi)
throws IOException, TezException, InterruptedException {
long startTime = System.currentTimeMillis();
long endTime = startTime + (clientTimeout * 1000);
Expand All @@ -172,7 +172,7 @@ protected DAGClientAMProtocolBlockingPB waitForProxy(long clientTimeout, Configu
* that proxy a shutdownSession was called.
*/
public boolean shutdownSession(Configuration conf, ApplicationId sessionAppId,
UserGroupInformation ugi) throws TezException, IOException, ServiceException {
UserGroupInformation ugi) throws TezException, IOException, ServiceException {
DAGClientAMProtocolBlockingPB proxy = getProxy(conf, sessionAppId, ugi);
if (proxy != null) {
ShutdownSessionRequestProto request = ShutdownSessionRequestProto.newBuilder().build();
Expand All @@ -183,7 +183,7 @@ public boolean shutdownSession(Configuration conf, ApplicationId sessionAppId,
}

protected DAGClientAMProtocolBlockingPB getProxy(Configuration conf, ApplicationId sessionAppId,
UserGroupInformation ugi) throws TezException, IOException {
UserGroupInformation ugi) throws TezException, IOException {
return TezClientUtils.getAMProxy(this, conf, sessionAppId, ugi);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Expand All @@ -25,5 +25,4 @@ public class TezApiVersionInfo extends VersionInfo {
public TezApiVersionInfo() {
super("tez-api");
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Expand Down
Loading